Improve logic to get the DDL statements as a part of the comparison. Fixes #5221

Fixed 23 issues related to schema diff.
This commit is contained in:
Akshay Joshi
2020-03-15 14:51:16 +05:30
parent 0b101d9efd
commit 44c0d76541
52 changed files with 951 additions and 921 deletions

View File

@@ -68,7 +68,6 @@ class SchemaDiffModule(PgAdminModule):
'schema_diff.connect_server',
'schema_diff.connect_database',
'schema_diff.get_server',
'schema_diff.generate_script',
'schema_diff.close'
]
@@ -452,7 +451,7 @@ def compare(trans_id, source_sid, source_did, source_scid,
for node_name, node_view in all_registered_nodes.items():
view = SchemaDiffRegistry.get_node_view(node_name)
if hasattr(view, 'compare'):
msg = "Comparing " + view.blueprint.COLLECTION_LABEL + " ..."
msg = "Comparing " + view.blueprint.COLLECTION_LABEL
diff_model_obj.set_comparison_info(msg, total_percent)
# Update the message and total percentage in session object
update_session_diff_transaction(trans_id, session_obj,
@@ -510,59 +509,6 @@ def poll(trans_id):
'diff_percentage': diff_percentage})
@blueprint.route(
'/generate_script/<int:trans_id>/',
methods=["POST"],
endpoint="generate_script"
)
def generate_script(trans_id):
"""This function will generate the scripts for the selected objects."""
data = request.form if request.form else json.loads(
request.data, encoding='utf-8'
)
status, error_msg, diff_model_obj, session_obj = \
check_transaction_status(trans_id)
if error_msg == gettext('Transaction ID not found in the session.'):
return make_json_response(success=0, errormsg=error_msg, status=404)
source_sid = int(data['source_sid'])
source_did = int(data['source_did'])
source_scid = int(data['source_scid'])
target_sid = int(data['target_sid'])
target_did = int(data['target_did'])
target_scid = int(data['target_scid'])
diff_ddl = ''
for d in data['sel_rows']:
node_type = d['node_type']
source_oid = int(d['source_oid'])
target_oid = int(d['target_oid'])
comp_status = d['comp_status']
view = SchemaDiffRegistry.get_node_view(node_type)
if view and hasattr(view, 'ddl_compare') and \
comp_status != SchemaDiffModel.COMPARISON_STATUS['identical']:
sql = view.ddl_compare(source_sid=source_sid,
source_did=source_did,
source_scid=source_scid,
target_sid=target_sid,
target_did=target_did,
target_scid=target_scid,
source_oid=source_oid,
target_oid=target_oid,
comp_status=comp_status,
generate_script=True)
diff_ddl += sql['diff_ddl'] + '\n\n'
return ajax_response(
status=200,
response={'diff_ddl': diff_ddl}
)
@blueprint.route(
'/ddl_compare/<int:trans_id>/<int:source_sid>/<int:source_did>/'
'<int:source_scid>/<int:target_sid>/<int:target_did>/<int:target_scid>/'
@@ -620,7 +566,11 @@ def check_version_compatibility(sid, tid):
tar_server = Server.query.filter_by(id=tid).first()
tar_manager = driver.connection_manager(tar_server.id)
tar_conn = tar_manager.connection()
if src_manager.server_type != tar_manager.server_type:
return False, gettext('Schema diff does not support the comparison '
'between Postgres Server and EDB Postgres '
'Advanced Server.')
if not (src_conn.connected() or src_conn.connected()):
return False, gettext('Server(s) disconnected.')

View File

@@ -15,8 +15,7 @@ from flask import render_template
from pgadmin.utils.driver import get_driver
from config import PG_DEFAULT_DRIVER
from pgadmin.utils.ajax import internal_server_error
from pgadmin.tools.schema_diff.directory_compare import compare_dictionaries,\
directory_diff
from pgadmin.tools.schema_diff.directory_compare import compare_dictionaries
from pgadmin.tools.schema_diff.model import SchemaDiffModel
@@ -68,10 +67,12 @@ class SchemaDiffObjectCompare:
'scid': kwargs.get('target_scid')
}
if 'source_tid' in kwargs:
source_params['tid'] = kwargs['source_tid']
if 'target_tid' in kwargs:
target_params['tid'] = kwargs['target_tid']
status, target_schema = self.get_schema(kwargs.get('target_sid'),
kwargs.get('target_did'),
kwargs.get('target_scid')
)
if not status:
return internal_server_error(errormsg=target_schema)
source = self.fetch_objects_to_compare(**source_params)
@@ -82,7 +83,8 @@ class SchemaDiffObjectCompare:
len(source) <= 0 and len(target) <= 0):
return None
return compare_dictionaries(source, target,
return compare_dictionaries(self, source_params, target_params,
target_schema, source, target,
self.node_type,
self.blueprint.COLLECTION_LABEL,
self.keys_to_ignore)
@@ -93,13 +95,6 @@ class SchemaDiffObjectCompare:
return the difference of SQL
"""
source = ''
target = ''
diff = ''
comp_status = kwargs.get('comp_status')
only_diff = False
generate_script = False
source_params = {'gid': 1,
'sid': kwargs.get('source_sid'),
'did': kwargs.get('source_did'),
@@ -114,98 +109,10 @@ class SchemaDiffObjectCompare:
'oid': kwargs.get('target_oid')
}
if 'source_tid' in kwargs:
source_params['tid'] = kwargs['source_tid']
only_diff = True
if 'target_tid' in kwargs:
target_params['tid'] = kwargs['target_tid']
only_diff = True
if 'generate_script' in kwargs and kwargs['generate_script']:
generate_script = True
source_params_adv = copy.deepcopy(source_params)
target_params_adv = copy.deepcopy(target_params)
del source_params_adv['gid']
del target_params_adv['gid']
status, target_schema = self.get_schema(kwargs.get('target_sid'),
kwargs.get('target_did'),
kwargs.get('target_scid')
)
if not status:
return internal_server_error(errormsg=target_schema)
if comp_status == SchemaDiffModel.COMPARISON_STATUS['source_only']:
if not generate_script:
source = self.get_sql_from_diff(**source_params)
source_params.update({
'diff_schema': target_schema
})
diff = self.get_sql_from_diff(**source_params)
elif comp_status == SchemaDiffModel.COMPARISON_STATUS['target_only']:
if not generate_script:
target = self.get_sql_from_diff(**target_params)
target_params.update(
{'drop_sql': True})
diff = self.get_sql_from_diff(**target_params)
elif comp_status == SchemaDiffModel.COMPARISON_STATUS['different']:
source = self.fetch_objects_to_compare(**source_params_adv)
target = self.fetch_objects_to_compare(**target_params_adv)
if not (source or target):
return None
diff_dict = directory_diff(source,
target,
ignore_keys=self.keys_to_ignore,
difference={}
)
diff_dict.update(self.parce_acl(source, target))
if not generate_script:
source = self.get_sql_from_diff(**source_params)
target = self.get_sql_from_diff(**target_params)
target_params.update(
{'data': diff_dict})
diff = self.get_sql_from_diff(**target_params)
else:
source = self.get_sql_from_diff(**source_params)
target = self.get_sql_from_diff(**target_params)
if only_diff:
return diff
source = self.get_sql_from_diff(**source_params)
target = self.get_sql_from_diff(**target_params)
return {'source_ddl': source,
'target_ddl': target,
'diff_ddl': diff
'diff_ddl': ''
}
@staticmethod
def parce_acl(source, target):
key = 'acl'
if 'datacl' in source:
key = 'datacl'
elif 'relacl' in source:
key = 'relacl'
tmp_source = source[key] if\
key in source and source[key] is not None else []
tmp_target = copy.deepcopy(target[key]) if\
key in target and target[key] is not None else []
diff = {'added': [], 'deleted': []}
for acl in tmp_source:
if acl in tmp_target:
tmp_target.remove(acl)
elif acl not in tmp_target:
diff['added'].append(acl)
diff['deleted'] = tmp_target
return {key: diff}

View File

@@ -15,11 +15,17 @@ from pgadmin.tools.schema_diff.model import SchemaDiffModel
count = 1
def compare_dictionaries(source_dict, target_dict, node, node_label,
def compare_dictionaries(view_object, source_params, target_params,
target_schema, source_dict, target_dict, node,
node_label,
ignore_keys=None):
"""
This function will compare the two dictionaries.
:param view_object: View Object
:param source_params: Source Parameters
:param target_params: Target Parameters
:param target_schema: Target Schema Name
:param source_dict: First Dictionary
:param target_dict: Second Dictionary
:param node: node type
@@ -36,18 +42,43 @@ def compare_dictionaries(source_dict, target_dict, node, node_label,
dict2_keys = set(dict2.keys())
intersect_keys = dict1_keys.intersection(dict2_keys)
# Add gid to the params
source_params['gid'] = target_params['gid'] = 1
# Keys that are available in source and missing in target.
source_only = []
added = dict1_keys - dict2_keys
global count
for item in added:
if node == 'table':
temp_src_params = copy.deepcopy(source_params)
temp_src_params['tid'] = source_dict[item]['oid']
temp_src_params['json_resp'] = False
source_ddl = \
view_object.get_sql_from_table_diff(**temp_src_params)
temp_src_params.update({
'diff_schema': target_schema
})
diff_ddl = view_object.get_sql_from_table_diff(**temp_src_params)
else:
temp_src_params = copy.deepcopy(source_params)
temp_src_params['oid'] = source_dict[item]['oid']
source_ddl = view_object.get_sql_from_diff(**temp_src_params)
temp_src_params.update({
'diff_schema': target_schema
})
diff_ddl = view_object.get_sql_from_diff(**temp_src_params)
source_only.append({
'id': count,
'type': node,
'label': node_label,
'title': item,
'oid': source_dict[item]['oid'],
'status': SchemaDiffModel.COMPARISON_STATUS['source_only']
'status': SchemaDiffModel.COMPARISON_STATUS['source_only'],
'source_ddl': source_ddl,
'target_ddl': '',
'diff_ddl': diff_ddl
})
count += 1
@@ -55,13 +86,34 @@ def compare_dictionaries(source_dict, target_dict, node, node_label,
# Keys that are available in target and missing in source.
removed = dict2_keys - dict1_keys
for item in removed:
if node == 'table':
temp_tgt_params = copy.deepcopy(target_params)
temp_tgt_params['tid'] = target_dict[item]['oid']
temp_tgt_params['json_resp'] = False
target_ddl = view_object.get_sql_from_table_diff(**temp_tgt_params)
if 'gid' in temp_tgt_params:
del temp_tgt_params['gid']
if 'json_resp' in temp_tgt_params:
del temp_tgt_params['json_resp']
diff_ddl = view_object.get_drop_sql(**temp_tgt_params)
else:
temp_tgt_params = copy.deepcopy(target_params)
temp_tgt_params['oid'] = target_dict[item]['oid']
target_ddl = view_object.get_sql_from_diff(**temp_tgt_params)
temp_tgt_params.update(
{'drop_sql': True})
diff_ddl = view_object.get_sql_from_diff(**temp_tgt_params)
target_only.append({
'id': count,
'type': node,
'label': node_label,
'title': item,
'oid': target_dict[item]['oid'],
'status': SchemaDiffModel.COMPARISON_STATUS['target_only']
'status': SchemaDiffModel.COMPARISON_STATUS['target_only'],
'source_ddl': '',
'target_ddl': target_ddl,
'diff_ddl': diff_ddl
})
count += 1
@@ -69,13 +121,6 @@ def compare_dictionaries(source_dict, target_dict, node, node_label,
identical = []
different = []
for key in intersect_keys:
# ignore the keys if available.
for ig_key in ignore_keys:
if ig_key in dict1[key]:
dict1[key].pop(ig_key)
if ig_key in dict2[key]:
dict2[key].pop(ig_key)
# Recursively Compare the two dictionary
if are_dictionaries_identical(dict1[key], dict2[key], ignore_keys):
identical.append({
@@ -89,6 +134,50 @@ def compare_dictionaries(source_dict, target_dict, node, node_label,
'status': SchemaDiffModel.COMPARISON_STATUS['identical']
})
else:
if node == 'table':
temp_src_params = copy.deepcopy(source_params)
temp_tgt_params = copy.deepcopy(target_params)
# Add submodules into the ignore keys so that directory
# difference won't include those in added, deleted and changed
sub_module = ['index', 'rule', 'trigger', 'compound_trigger']
temp_ignore_keys = view_object.keys_to_ignore + sub_module
diff_dict = directory_diff(
dict1[key], dict2[key],
ignore_keys=temp_ignore_keys,
difference={}
)
diff_dict.update(parce_acl(dict1[key], dict2[key]))
temp_src_params['tid'] = source_dict[key]['oid']
temp_tgt_params['tid'] = target_dict[key]['oid']
temp_src_params['json_resp'] = \
temp_tgt_params['json_resp'] = False
source_ddl = \
view_object.get_sql_from_table_diff(**temp_src_params)
target_ddl = \
view_object.get_sql_from_table_diff(**temp_tgt_params)
diff_ddl = view_object.get_sql_from_submodule_diff(
temp_src_params, temp_tgt_params, target_schema,
dict1[key], dict2[key], diff_dict)
else:
temp_src_params = copy.deepcopy(source_params)
temp_tgt_params = copy.deepcopy(target_params)
diff_dict = directory_diff(
dict1[key], dict2[key],
ignore_keys=view_object.keys_to_ignore, difference={}
)
diff_dict.update(parce_acl(dict1[key], dict2[key]))
temp_src_params['oid'] = source_dict[key]['oid']
temp_tgt_params['oid'] = target_dict[key]['oid']
source_ddl = view_object.get_sql_from_diff(**temp_src_params)
target_ddl = view_object.get_sql_from_diff(**temp_tgt_params)
temp_tgt_params.update(
{'data': diff_dict})
diff_ddl = view_object.get_sql_from_diff(**temp_tgt_params)
different.append({
'id': count,
'type': node,
@@ -97,7 +186,10 @@ def compare_dictionaries(source_dict, target_dict, node, node_label,
'oid': source_dict[key]['oid'],
'source_oid': source_dict[key]['oid'],
'target_oid': target_dict[key]['oid'],
'status': SchemaDiffModel.COMPARISON_STATUS['different']
'status': SchemaDiffModel.COMPARISON_STATUS['different'],
'source_ddl': source_ddl,
'target_ddl': target_ddl,
'diff_ddl': diff_ddl
})
count += 1
@@ -143,13 +235,6 @@ def are_dictionaries_identical(source_dict, target_dict, ignore_keys):
src_keys = set(source_dict.keys())
tar_keys = set(target_dict.keys())
# ignore the keys if available.
for ig_key in ignore_keys:
if ig_key in src_keys:
source_dict.pop(ig_key)
if ig_key in target_dict:
target_dict.pop(ig_key)
# Keys that are available in source and missing in target.
src_only = src_keys - tar_keys
# Keys that are available in target and missing in source.
@@ -167,6 +252,10 @@ def are_dictionaries_identical(source_dict, target_dict, ignore_keys):
return False
for key in source_dict.keys():
# Continue if key is available in ignore_keys
if key in ignore_keys:
continue
if type(source_dict[key]) is dict:
if not are_dictionaries_identical(source_dict[key],
target_dict[key], ignore_keys):
@@ -235,33 +324,32 @@ def directory_diff(source_dict, target_dict, ignore_keys=[], difference={}):
# TODO
pass
elif type(source) is dict:
if 'name' in source or 'colname' in source:
if type(target_dict[key]) is list and len(
target_dict[key]) > 0:
tmp = None
tmp_target = copy.deepcopy(target_dict[key])
for item in tmp_target:
if (
'name' in item and
item['name'] == source['name']
) or (
'colname' in item and
item['colname'] == source[
'colname']
):
tmp = copy.deepcopy(item)
if tmp and source != tmp:
updated.append(copy.deepcopy(source))
tmp_target.remove(tmp)
elif tmp and source == tmp:
tmp_target.remove(tmp)
elif tmp is None:
tmp_key_array = ['name', 'colname', 'argid']
for tmp_key in tmp_key_array:
if tmp_key in source:
if type(target_dict[key]) is list and \
len(target_dict[key]) > 0:
tmp = None
tmp_target = \
copy.deepcopy(target_dict[key])
for item in tmp_target:
if tmp_key in item and \
item[tmp_key] == \
source[tmp_key]:
tmp = copy.deepcopy(item)
if tmp and source != tmp:
updated.append(copy.deepcopy(source))
tmp_target.remove(tmp)
elif tmp and source == tmp:
tmp_target.remove(tmp)
elif tmp is None:
added.append(source)
else:
added.append(source)
else:
added.append(source)
difference[key] = {}
difference[key]['added'] = added
difference[key]['changed'] = updated
difference[key] = {}
difference[key]['added'] = added
difference[key]['changed'] = updated
elif target_dict[key] is None or \
(type(target_dict[key]) is list and
len(target_dict[key]) < index and
@@ -271,7 +359,7 @@ def directory_diff(source_dict, target_dict, ignore_keys=[], difference={}):
len(target_dict[key]) > index:
difference[key] = source
else:
target_dict[key] = source_dict[key]
difference[key] = source_dict[key]
if type(source) is dict and tmp_target and key in tmp_target and \
tmp_target[key] and len(tmp_target[key]) > 0:
@@ -286,6 +374,34 @@ def directory_diff(source_dict, target_dict, ignore_keys=[], difference={}):
else:
if source_dict[key] != target_dict[key]:
difference[key] = source_dict[key]
if (key == 'comment' or key == 'description') and \
source_dict[key] is None:
difference[key] = ''
else:
difference[key] = source_dict[key]
return difference
def parce_acl(source, target):
key = 'acl'
if 'datacl' in source:
key = 'datacl'
elif 'relacl' in source:
key = 'relacl'
tmp_source = source[key] if\
key in source and source[key] is not None else []
tmp_target = copy.deepcopy(target[key]) if\
key in target and target[key] is not None else []
diff = {'added': [], 'deleted': []}
for acl in tmp_source:
if acl in tmp_target:
tmp_target.remove(acl)
elif acl not in tmp_target:
diff['added'].append(acl)
diff['deleted'] = tmp_target
return {key: diff}

View File

@@ -63,6 +63,8 @@ let SchemaDiffSqlControl =
},
render: function() {
let obj = Backform.SqlFieldControl.prototype.render.apply(this, arguments);
obj.sqlCtrl.setOption('readOnly', true);
if(this.$el.find('.ddl-copy')) this.$el.find('.ddl-copy').on('click', this.copyData);
return obj;
},

View File

@@ -197,7 +197,6 @@ export default class SchemaDiffUI {
baseServerUrl = url_for('schema_diff.get_server', {'sid': self.selection['target_sid'],
'did': self.selection['target_did']}),
sel_rows = self.grid ? self.grid.getSelectedRows() : [],
sel_rows_data = [],
url_params = self.selection,
generated_script = undefined,
open_query_tool,
@@ -261,45 +260,16 @@ export default class SchemaDiffUI {
};
if (sel_rows.length > 0) {
let script_body = '';
for (var row = 0; row < sel_rows.length; row++) {
let data = self.grid.getData().getItem(sel_rows[row]);
if (data.type) {
let tmp_data = {
'node_type': data.type,
'source_oid': parseInt(data.oid, 10),
'target_oid': parseInt(data.oid, 10),
'comp_status': data.status,
};
if(data.status && (data.status.toLowerCase() == 'different' || data.status.toLowerCase() == 'identical')) {
tmp_data['target_oid'] = data.target_oid;
}
sel_rows_data.push(tmp_data);
if(!_.isUndefined(data.diff_ddl)) {
script_body += data.diff_ddl + '\n\n';
}
}
url_params['sel_rows'] = sel_rows_data;
let baseUrl = url_for('schema_diff.generate_script', {'trans_id': self.trans_id});
$.ajax({
url: baseUrl,
method: 'POST',
dataType: 'json',
contentType: 'application/json',
data: JSON.stringify(url_params),
})
.done(function (res) {
if (res) {
generated_script = script_header + 'BEGIN;' + '\n' + res.diff_ddl + '\n' + 'END;';
}
open_query_tool();
})
.fail(function (xhr) {
self.raise_error_on_fail(gettext('Generate script error'), xhr);
$('#diff_fetching_data').addClass('d-none');
});
generated_script = script_header + 'BEGIN;' + '\n' + script_body + 'END;';
open_query_tool();
} else if (!_.isUndefined(self.model.get('diff_ddl'))) {
open_query_tool();
}
@@ -448,6 +418,7 @@ export default class SchemaDiffUI {
render_grid_data(data) {
var self = this;
self.grid.setSelectedRows([]);
data.sort((a, b) => (a.label > b.label) ? 1 : (a.label === b.label) ? ((a.title > b.title) ? 1 : -1) : -1);
self.dataView.beginUpdate();
self.dataView.setItems(data);
self.dataView.setFilter(self.filter.bind(self));
@@ -489,8 +460,13 @@ export default class SchemaDiffUI {
contentType: 'application/json',
})
.done(function (res) {
let msg = res.data.compare_msg + res.data.diff_percentage + '% completed';
$('#diff_fetching_data').find('.schema-diff-busy-text').text(msg);
let msg = res.data.compare_msg;
if (res.data.diff_percentage != 100) {
msg = msg + ' (this may take a few minutes)...';
}
msg = msg + '<br>'+ res.data.diff_percentage + '% completed.';
$('#diff_fetching_data').find('.schema-diff-busy-text').html(msg);
})
.fail(function (xhr) {
self.raise_error_on_fail(gettext('Poll error'), xhr);
@@ -528,37 +504,44 @@ export default class SchemaDiffUI {
'diff_ddl': undefined,
});
var url_params = self.selection;
if(data.status && (data.status.toLowerCase() == 'different' || data.status.toLowerCase() == 'identical')) {
if(data.status && data.status.toLowerCase() == 'identical') {
var url_params = self.selection;
target_oid = data.target_oid;
url_params['trans_id'] = self.trans_id;
url_params['source_oid'] = source_oid;
url_params['target_oid'] = target_oid;
url_params['comp_status'] = data.status;
url_params['node_type'] = node_type;
_.each(url_params, function(key, val) {
url_params[key] = parseInt(val, 10);
});
$('#ddl_comp_fetching_data').removeClass('d-none');
var baseUrl = url_for('schema_diff.ddl_compare', url_params);
self.model.url = baseUrl;
self.model.fetch({
success: function() {
self.footer.render();
$('#ddl_comp_fetching_data').addClass('d-none');
},
error: function() {
self.footer.render();
$('#ddl_comp_fetching_data').addClass('d-none');
},
});
} else {
self.model.set({
'source_ddl': data.source_ddl,
'target_ddl': data.target_ddl,
'diff_ddl': data.diff_ddl,
});
self.footer.render();
}
url_params['trans_id'] = self.trans_id;
url_params['source_oid'] = source_oid;
url_params['target_oid'] = target_oid;
url_params['comp_status'] = data.status;
url_params['node_type'] = node_type;
_.each(url_params, function(key, val) {
url_params[key] = parseInt(val, 10);
});
$('#ddl_comp_fetching_data').removeClass('d-none');
var baseUrl = url_for('schema_diff.ddl_compare', url_params);
self.model.url = baseUrl;
self.model.fetch({
success: function() {
self.footer.render();
$('#ddl_comp_fetching_data').addClass('d-none');
},
error: function() {
self.footer.render();
$('#ddl_comp_fetching_data').addClass('d-none');
},
});
}
render() {

View File

@@ -142,16 +142,10 @@ class SchemaDiffTestCase(BaseTestGenerator):
file_obj = open(diff_file, 'a')
for diff in response_data['data']:
if diff['type'] in self.nodes:
src_obj_oid = tar_obj_oid = None
if diff['status'] == 'Source Only' or\
diff['status'] == 'Target Only':
src_obj_oid = tar_obj_oid = diff['oid']
elif diff['status'] == 'Different':
src_obj_oid = diff['source_oid']
tar_obj_oid = diff['target_oid']
if src_obj_oid is not None:
if diff['type'] in self.nodes and diff['status'] == 'Identical':
src_obj_oid = diff['source_oid']
tar_obj_oid = diff['target_oid']
if src_obj_oid is not None and tar_obj_oid is not None:
url = 'schema_diff/ddl_compare/{0}/{1}/{2}/{3}/{4}/{5}/' \
'{6}/{7}/{8}/{9}/{10}/'.format(self.trans_id,
self.server_id,