Added support of Domain, Domain Constraints and Types to the Schema Diff. Fixes #5262

This commit is contained in:
Akshay Joshi
2020-03-26 14:30:09 +05:30
parent 4c0b229b14
commit 94a76cc9e0
20 changed files with 1968 additions and 29 deletions

View File

@@ -179,6 +179,8 @@ class DomainView(PGChildNodeView, DataTypeReader, SchemaDiffObjectCompare):
'compare': [{'get': 'compare'}, {'get': 'compare'}]
})
keys_to_ignore = ['oid', 'basensp', 'conoid', 'nspname']
def validate_request(f):
"""
Works as a decorator.
@@ -584,7 +586,7 @@ AND relkind != 'c'))"""
)
@check_precondition
def delete(self, gid, sid, did, scid, doid=None):
def delete(self, gid, sid, did, scid, doid=None, only_sql=False):
"""
Drops the Domain object.
@@ -594,6 +596,7 @@ AND relkind != 'c'))"""
did: Database Id
scid: Schema Id
doid: Domain Id
only_sql: Return only sql if True
"""
if doid is None:
data = request.form if request.form else json.loads(
@@ -602,7 +605,7 @@ AND relkind != 'c'))"""
else:
data = {'ids': [doid]}
if self.cmd == 'delete':
if self.cmd == 'delete' or only_sql:
# This is a cascade operation
cascade = True
else:
@@ -634,6 +637,11 @@ AND relkind != 'c'))"""
SQL = render_template("/".join([self.template_path,
'delete.sql']),
name=name, basensp=basensp, cascade=cascade)
# Used for schema diff tool
if only_sql:
return SQL
status, res = self.conn.execute_scalar(SQL)
if not status:
return internal_server_error(errormsg=res)
@@ -684,7 +692,8 @@ AND relkind != 'c'))"""
)
@check_precondition
def sql(self, gid, sid, did, scid, doid=None, return_ajax_response=True):
def sql(self, gid, sid, did, scid, doid=None, diff_schema=None,
json_resp=True):
"""
Returns the SQL for the Domain object.
@@ -694,7 +703,8 @@ AND relkind != 'c'))"""
did: Database Id
scid: Schema Id
doid: Domain Id
return_ajax_response:
diff_schema: Target Schema for schema diff
json_resp: True then return json response
"""
SQL = render_template("/".join([self.template_path,
@@ -710,6 +720,9 @@ AND relkind != 'c'))"""
data = res['rows'][0]
if diff_schema:
data['basensp'] = diff_schema
# Get Type Length and Precision
data.update(self._parse_type(data['fulltype']))
@@ -737,7 +750,7 @@ AND relkind != 'c'))"""
""".format(self.qtIdent(self.conn, data['basensp'], data['name']))
SQL = sql_header + SQL
if not return_ajax_response:
if not json_resp:
return SQL.strip('\n')
return ajax_response(response=SQL.strip('\n'))
@@ -780,7 +793,7 @@ AND relkind != 'c'))"""
except Exception as e:
return internal_server_error(errormsg=str(e))
def get_sql(self, gid, sid, data, scid, doid=None):
def get_sql(self, gid, sid, data, scid, doid=None, is_schema_diff=False):
"""
Generates the SQL statements to create/update the Domain.
@@ -790,6 +803,7 @@ AND relkind != 'c'))"""
did: Database Id
scid: Schema Id
doid: Domain Id
is_schema_diff: True is function gets called from schema diff
"""
if doid is not None:
@@ -821,9 +835,19 @@ AND relkind != 'c'))"""
old_data['constraints'] = con_data
SQL = render_template(
"/".join([self.template_path, 'update.sql']),
data=data, o_data=old_data)
# If fulltype or basetype or collname is changed while comparing
# two schemas then we need to drop domain and recreate it
if 'fulltype' in data or 'basetype' in data or 'collname' in data:
SQL = render_template(
"/".join([self.template_path, 'domain_schema_diff.sql']),
data=data, o_data=old_data)
else:
if is_schema_diff:
data['is_schema_diff'] = True
SQL = render_template(
"/".join([self.template_path, 'update.sql']),
data=data, o_data=old_data)
return SQL.strip('\n'), data['name'] if 'name' in data else \
old_data['name']
else:
@@ -892,18 +916,43 @@ AND relkind != 'c'))"""
status, data = self._fetch_properties(did, scid, row['oid'])
if status:
if 'constraints' in data and len(data['constraints']) > 0:
for item in data['constraints']:
# Remove keys that should not be the part
# of comparision.
if 'conoid' in item:
item.pop('conoid')
if 'nspname' in item:
item.pop('nspname')
res[row['name']] = data
return res
def get_sql_from_diff(self, gid, sid, did, scid, oid, data=None,
diff_schema=None, drop_sql=False):
"""
This function is used to get the DDL/DML statements.
:param gid: Group ID
:param sid: Serve ID
:param did: Database ID
:param scid: Schema ID
:param oid: Collation ID
:param data: Difference data
:param diff_schema: Target Schema
:param drop_sql: True if need to drop the domains
:return:
"""
sql = ''
if data:
if diff_schema:
data['schema'] = diff_schema
sql, name = self.get_sql(gid=gid, sid=sid, scid=scid,
data=data, doid=oid,
is_schema_diff=True)
else:
if drop_sql:
sql = self.delete(gid=gid, sid=sid, did=did,
scid=scid, doid=oid, only_sql=True)
elif diff_schema:
sql = self.sql(gid=gid, sid=sid, did=did, scid=scid, doid=oid,
diff_schema=diff_schema, json_resp=False)
else:
sql = self.sql(gid=gid, sid=sid, did=did, scid=scid, doid=oid,
json_resp=False)
return sql
SchemaDiffRegistry(blueprint.node_type, DomainView)
DomainView.register_node_view(blueprint)

View File

@@ -0,0 +1,44 @@
{% import 'macros/schemas/security.macros' as SECLABEL %}
-- WARNING:
-- We have found the difference in either of datatype or collation,
-- so we need to drop the existing domain first and re-create it.
DROP DOMAIN {{ conn|qtIdent(o_data.basensp, o_data.name) }};
CREATE DOMAIN {{ conn|qtIdent(o_data.basensp, o_data.name) }}
AS {% if data.fulltype %}{{ data.fulltype }}{% else %}{{ o_data.fulltype }}{% endif %}{% if data.collname and data.collname != "pg_catalog.\"default\"" %}
COLLATE {{ data.collname }}{% endif %}{% if data.typdefault %}
DEFAULT {{ data.typdefault }}{% endif %}{% if data.typnotnull %}
NOT NULL{% endif %};
{% if data.owner %}
ALTER DOMAIN {{ conn|qtIdent(o_data.basensp, o_data.name) }} OWNER TO {{ conn|qtIdent(data.owner) }};
{% endif %}
{% if data.constraints %}
{% for c in data.constraints.added %}{% if c.conname and c.consrc %}
ALTER DOMAIN {{ conn|qtIdent(o_data.basensp, o_data.name) }}
ADD CONSTRAINT {{ conn|qtIdent(c.conname) }} CHECK ({{ c.consrc }}){% if not c.convalidated %} NOT VALID{% endif %}{% endif -%};
{% endfor -%}
{% for c in data.constraints.changed %}{% if c.conname and c.consrc %}
ALTER DOMAIN {{ conn|qtIdent(o_data.basensp, o_data.name) }}
ADD CONSTRAINT {{ conn|qtIdent(c.conname) }} CHECK ({{ c.consrc }}){% if not c.convalidated %} NOT VALID{% endif %}{% endif -%};
{% endfor -%}
{% endif %}
{% if data.description %}
COMMENT ON DOMAIN {{ conn|qtIdent(o_data.basensp, o_data.name) }}
IS '{{ data.description }}';{% endif -%}
{% if data.seclabels %}
{% for r in data.seclabels %}
{% if r.label and r.provider %}
{{ SECLABEL.SET(conn, 'DOMAIN', data.name, r.provider, r.label, data.basensp) }}{% endif -%}
{% endfor -%}
{% endif -%}

View File

@@ -20,7 +20,7 @@ ALTER DOMAIN {{ conn|qtIdent(o_data.basensp, name) }}
ALTER DOMAIN {{ conn|qtIdent(o_data.basensp, name) }}
SET DEFAULT {{ data.typdefault }};
{% elif data.typdefault == '' and o_data.typdefault %}
{% elif (data.typdefault == '' or data.typdefault == None) and data.typdefault != o_data.typdefault %}
ALTER DOMAIN {{ conn|qtIdent(o_data.basensp, name) }}
DROP DEFAULT;
@@ -34,6 +34,19 @@ ALTER DOMAIN {{ conn|qtIdent(o_data.basensp, name) }}
ALTER DOMAIN {{ conn|qtIdent(o_data.basensp, name) }}
DROP CONSTRAINT {{ conn|qtIdent(o_data['constraints'][c.conoid]['conname']) }};
{% endfor -%}
{% if data.is_schema_diff is defined and data.is_schema_diff %}
{% for c in data.constraints.changed %}
ALTER DOMAIN {{ conn|qtIdent(o_data.basensp, name) }}
DROP CONSTRAINT {{ conn|qtIdent(c.conname) }};
ALTER DOMAIN {{ conn|qtIdent(o_data.basensp, name) }}
ADD CONSTRAINT {{ conn|qtIdent(c.conname) }} CHECK ({{ c.consrc }}){% if not c.convalidated %} NOT VALID{% endif %}{% if c.connoinherit %} NO INHERIT{% endif -%};
{% if c.description is defined and c.description != '' %}
COMMENT ON CONSTRAINT {{ conn|qtIdent(c.conname) }} ON DOMAIN {{ conn|qtIdent(o_data.basensp, name) }}
IS {{ c.description|qtLiteral }};{% endif %}
{% endfor -%}
{% else %}
{% for c in data.constraints.changed %}
{% if c.conname and c.conname !=o_data['constraints'][c.conoid]['conname'] %}
@@ -46,6 +59,7 @@ ALTER DOMAIN {{ conn|qtIdent(o_data.basensp, name) }}
VALIDATE CONSTRAINT {{ conn|qtIdent(c.conname) }};
{% endif %}
{% endfor -%}
{% endif %}
{% for c in data.constraints.added %}
{% if c.conname and c.consrc %}

View File

@@ -219,6 +219,9 @@ class TypeView(PGChildNodeView, DataTypeReader, SchemaDiffObjectCompare):
{'get': 'get_external_functions_list'}]
})
keys_to_ignore = ['oid', 'typnamespace', 'typrelid', 'typarray', 'alias',
'schema']
def check_precondition(f):
"""
This function will behave as a decorator which will checks
@@ -1071,7 +1074,7 @@ class TypeView(PGChildNodeView, DataTypeReader, SchemaDiffObjectCompare):
return internal_server_error(errormsg=str(e))
@check_precondition
def delete(self, gid, sid, did, scid, tid=None):
def delete(self, gid, sid, did, scid, tid=None, only_sql=False):
"""
This function will updates existing the type object
@@ -1081,6 +1084,7 @@ class TypeView(PGChildNodeView, DataTypeReader, SchemaDiffObjectCompare):
did: Database ID
scid: Schema ID
tid: Type ID
only_sql: Return only sql if True
"""
if tid is None:
data = request.form if request.form else json.loads(
@@ -1090,7 +1094,7 @@ class TypeView(PGChildNodeView, DataTypeReader, SchemaDiffObjectCompare):
data = {'ids': [tid]}
# Below will decide if it's simple drop or drop with cascade call
if self.cmd == 'delete':
if self.cmd == 'delete' or only_sql:
# This is a cascade operation
cascade = True
else:
@@ -1128,6 +1132,11 @@ class TypeView(PGChildNodeView, DataTypeReader, SchemaDiffObjectCompare):
data=data,
cascade=cascade,
conn=self.conn)
# Used for schema diff tool
if only_sql:
return SQL
status, res = self.conn.execute_scalar(SQL)
if not status:
return internal_server_error(errormsg=res)
@@ -1279,10 +1288,19 @@ class TypeView(PGChildNodeView, DataTypeReader, SchemaDiffObjectCompare):
old_data.update(self.additional_properties(old_data, tid))
old_data = self._convert_for_sql(old_data)
SQL = render_template(
"/".join([self.template_path, 'update.sql']),
data=data, o_data=old_data, conn=self.conn
)
# If typname or collname is changed while comparing
# two schemas then we need to drop type and recreate it
if 'typtype' in data or 'typname' in data or 'collname' in data\
or 'typinput' in data or 'typoutput' in data:
SQL = render_template(
"/".join([self.template_path, 'type_schema_diff.sql']),
data=data, o_data=old_data, conn=self.conn
)
else:
SQL = render_template(
"/".join([self.template_path, 'update.sql']),
data=data, o_data=old_data, conn=self.conn
)
else:
required_args = [
'name',
@@ -1331,7 +1349,8 @@ class TypeView(PGChildNodeView, DataTypeReader, SchemaDiffObjectCompare):
return SQL, data['name'] if 'name' in data else old_data['name']
@check_precondition
def sql(self, gid, sid, did, scid, tid):
def sql(self, gid, sid, did, scid, tid, diff_schema=None,
json_resp=True):
"""
This function will generates reverse engineered sql for type object
@@ -1341,6 +1360,8 @@ class TypeView(PGChildNodeView, DataTypeReader, SchemaDiffObjectCompare):
did: Database ID
scid: Schema ID
tid: Type ID
diff_schema: Target Schema for schema diff
json_resp: True then return json response
"""
SQL = render_template(
"/".join([self.template_path,
@@ -1359,6 +1380,9 @@ class TypeView(PGChildNodeView, DataTypeReader, SchemaDiffObjectCompare):
# Making copy of output for future use
data = dict(res['rows'][0])
if diff_schema:
data['schema'] = diff_schema
SQL = render_template("/".join([self.template_path, 'acl.sql']),
scid=scid, tid=tid)
status, acl = self.conn.execute_dict(SQL)
@@ -1401,6 +1425,9 @@ class TypeView(PGChildNodeView, DataTypeReader, SchemaDiffObjectCompare):
data=data, conn=self.conn)
SQL = sql_header + '\n\n' + SQL
if not json_resp:
return SQL.strip('\n')
return ajax_response(response=SQL)
@check_precondition
@@ -1473,5 +1500,38 @@ class TypeView(PGChildNodeView, DataTypeReader, SchemaDiffObjectCompare):
return res
def get_sql_from_diff(self, gid, sid, did, scid, oid, data=None,
diff_schema=None, drop_sql=False):
"""
This function is used to get the DDL/DML statements.
:param gid: Group ID
:param sid: Serve ID
:param did: Database ID
:param scid: Schema ID
:param oid: Collation ID
:param data: Difference data
:param diff_schema: Target Schema
:param drop_sql: True if need to drop the types
:return:
"""
sql = ''
if data:
if diff_schema:
data['schema'] = diff_schema
sql, name = self.get_sql(gid=gid, sid=sid, scid=scid,
data=data, tid=oid)
else:
if drop_sql:
sql = self.delete(gid=gid, sid=sid, did=did,
scid=scid, tid=oid, only_sql=True)
elif diff_schema:
sql = self.sql(gid=gid, sid=sid, did=did, scid=scid, tid=oid,
diff_schema=diff_schema, json_resp=False)
else:
sql = self.sql(gid=gid, sid=sid, did=did, scid=scid, tid=oid,
json_resp=False)
return sql
SchemaDiffRegistry(blueprint.node_type, TypeView)
TypeView.register_node_view(blueprint)

View File

@@ -26,12 +26,14 @@ FROM pg_enum
{# The SQL given below will fetch range type#}
{% if type == 'r' %}
SELECT rngsubtype, st.typname,
rngcollation, col.collname,
rngcollation,
CASE WHEN n.nspname IS NOT NULL THEN concat(quote_ident(n.nspname), '.', quote_ident(col.collname)) ELSE col.collname END AS collname,
rngsubopc, opc.opcname,
rngcanonical, rngsubdiff
FROM pg_range
LEFT JOIN pg_type st ON st.oid=rngsubtype
LEFT JOIN pg_collation col ON col.oid=rngcollation
LEFT JOIN pg_namespace n ON col.collnamespace=n.oid
LEFT JOIN pg_opclass opc ON opc.oid=rngsubopc
WHERE rngtypid={{tid}}::oid;
{% endif %}

View File

@@ -0,0 +1,105 @@
{% import 'macros/schemas/security.macros' as SECLABEL %}
{% import 'macros/schemas/privilege.macros' as PRIVILEGE %}
{% import 'types/macros/get_full_type_sql_format.macros' as GET_TYPE %}
-- WARNING:
-- We have found the difference in either of Type or SubType or Collation,
-- so we need to drop the existing type first and re-create it.
DROP TYPE {{ conn|qtIdent(o_data.schema, o_data.name) }} CASCADE;
{## If user selected shell type then just create type template ##}
{% if data and data.typtype == 'p' %}
CREATE TYPE {{ conn|qtIdent(o_data.schema, o_data.name) }};
{% endif %}
{### Composite Type ###}
{% if data and data.typtype == 'c' %}
{% if data.composite %}{% set typinput = data.typinput %}{% elif o_data.typinput %}{% set typinput = o_data.typinput %}{% endif %}
CREATE TYPE {% if o_data.schema %}{{ conn|qtIdent(o_data.schema, o_data.name) }}{% else %}{{ conn|qtIdent(o_data.name) }}{% endif %} AS
({{"\n\t"}}{% if data.composite.added %}{% for d in data.composite.added %}{% if loop.index != 1 %},{{"\n\t"}}{% endif %}{{ conn|qtIdent(d.member_name) }} {% if is_sql %}{{ d.fulltype }}{% else %}{{ GET_TYPE.CREATE_TYPE_SQL(conn, d.cltype, d.tlength, d.precision, d.hasSqrBracket) }}{% endif %}{% if d.collation %} COLLATE {{d.collation}}{% endif %}{% endfor %}{% endif %}{{"\n"}});
{% endif %}
{### Enum Type ###}
{% if data and data.typtype == 'e' %}
CREATE TYPE {% if o_data.schema %}{{ conn|qtIdent(o_data.schema, o_data.name) }}{% else %}{{ conn|qtIdent(o_data.name) }}{% endif %} AS ENUM
({% for e in data.enum.added %}{% if loop.index != 1 %}, {% endif %}{{ e.label|qtLiteral }}{% endfor %});
{% endif %}
{### Range Type ###}
{% if data and (data.typtype == 'r' or (data.typtype is not defined and o_data.typtype == 'r')) %}
{% if data.typname %}{% set typname = data.typname %}{% elif o_data.typname %}{% set typname = o_data.typname %}{% endif %}
CREATE TYPE {% if o_data.schema %}{{ conn|qtIdent(o_data.schema, o_data.name) }}{% else %}{{ conn|qtIdent(o_data.name) }}{% endif %} AS RANGE
(
{% if typname %}SUBTYPE={{ conn|qtTypeIdent(typname) }}{% endif %}{% if data.collname %},
COLLATION = {{ data.collname }}{% endif %}{% if data.opcname %},
SUBTYPE_OPCLASS = {{ data.opcname }}{% endif %}{% if data.rngcanonical %},
CANONICAL = {{ data.rngcanonical }}{% endif %}{% if data.rngsubdiff %},
SUBTYPE_DIFF = {{ data.rngsubdiff }}{% endif %}
);
{% endif %}
{### External Type ###}
{% if data and (data.typtype == 'b' or (data.typtype is not defined and o_data.typtype == 'b')) %}
{% if data.typinput %}{% set typinput = data.typinput %}{% elif o_data.typinput %}{% set typinput = o_data.typinput %}{% endif %}
{% if data.typoutput %}{% set typoutput = data.typoutput %}{% elif o_data.typoutput %}{% set typoutput = o_data.typoutput %}{% endif %}
CREATE TYPE {% if o_data.schema %}{{ conn|qtIdent(o_data.schema, o_data.name) }}{% else %}{{ conn|qtIdent(o_data.name) }}{% endif %}
(
{% if typinput %}INPUT = {{ typinput }}{% endif %}{% if typoutput %},
OUTPUT = {{ typoutput }}{% endif %}{% if data.typreceive %},
RECEIVE = {{data.typreceive}}{% endif %}{% if data.typsend %},
SEND = {{data.typsend}}{% endif %}{% if data.typmodin %},
TYPMOD_IN = {{data.typmodin}}{% endif %}{% if data.typmodout %},
TYPMOD_OUT = {{data.typmodout}}{% endif %}{% if data.typanalyze %},
ANALYZE = {{data.typanalyze}}{% endif %}{% if data.typlen %},
INTERNALLENGTH = {{data.typlen}}{% endif %}{% if data.typbyval %},
PASSEDBYVALUE{% endif %}{% if data.typalign %},
ALIGNMENT = {{data.typalign}}{% endif %}{% if data.typstorage %},
STORAGE = {{data.typstorage}}{% endif %}{% if data.typcategory %},
CATEGORY = {{data.typcategory|qtLiteral}}{% endif %}{% if data.typispreferred %},
PREFERRED = {{data.typispreferred}}{% endif %}{% if data.typdefault %},
DEFAULT = {{data.typdefault|qtLiteral}}{% endif %}{% if data.element %},
ELEMENT = {{data.element}}{% endif %}{% if data.typdelim %},
DELIMITER = {{data.typdelim|qtLiteral}}{% endif %}{% if data.is_collatable %},
COLLATABLE = {{data.is_collatable}}{% endif %}
);
{% endif %}
{### Type Owner ###}
{% if data and data.typeowner %}
ALTER TYPE {% if o_data.schema %}{{ conn|qtIdent(o_data.schema, o_data.name) }}{% else %}{{ conn|qtIdent(o_data.name) }}{% endif %}
OWNER TO {{ conn|qtIdent(data.typeowner) }};
{% endif %}
{### Type Comments ###}
{% if data and data.description %}
COMMENT ON TYPE {% if o_data.schema %}{{ conn|qtIdent(o_data.schema, o_data.name) }}{% else %}{{ conn|qtIdent(o_data.name) }}{% endif %}
IS {{data.description|qtLiteral}};
{% endif %}
{### ACL ###}
{% if data.typacl and data.typacl|length > 0 %}
{% if 'deleted' in data.typacl %}
{% for priv in data.typacl.deleted %}
{{ PRIVILEGE.UNSETALL(conn, 'TYPE', priv.grantee, o_data.name, o_data.schema) }}
{% endfor %}
{% endif %}
{% if 'changed' in data.typacl %}
{% for priv in data.typacl.changed %}
{{ PRIVILEGE.UNSETALL(conn, 'TYPE', priv.grantee, o_data.name, o_data.schema) }}
{{ PRIVILEGE.SET(conn, 'TYPE', priv.grantee, o_data.name, priv.without_grant, priv.with_grant, o_data.schema) }}
{% endfor %}
{% endif %}
{% if 'added' in data.typacl %}
{% for priv in data.typacl.added %}
{{ PRIVILEGE.SET(conn, 'TYPE', priv.grantee, o_data.name, priv.without_grant, priv.with_grant, o_data.schema) }}
{% endfor %}
{% endif %}
{% endif %}
{### Security Lables ###}
{% if data.seclabels %}
{% for r in data.seclabels %}
{% if r.provider and r.label %}
{{ SECLABEL.SET(conn, 'TYPE', o_data.name, r.provider, r.label, o_data.schema) }}
{% endif %}
{% endfor %}
{% endif %}