mirror of
https://github.com/pgadmin-org/pgadmin4.git
synced 2026-10-28 23:50:04 -05:00
Drop the scaffolding default partition after Schema Diff rebuilds a partitioned table. #10301 (#10316)
The temporary default partition is dropped when empty and is not created at all for hash-partitioned tables, which cannot have one.
This commit is contained in:
1 parent
c41b606795
commit
6302a70316
6 files changed
+523
-13
No files matched your search
+27
-2
@@ -507,11 +507,36 @@ class PartitionsView(BaseTableView, DataTypeReader, SchemaDiffObjectCompare):
|
||||
target_data['orig_name'] = target_data['name']
|
||||
target_data['name'] = 'temp_partitioned_{0}'.format(
|
||||
secrets.choice(range(1, 9999999)))
|
||||
|
||||
# If the source table already has a default partition of its own,
|
||||
# it will be (re)created below via source_partitions, so we
|
||||
# must not scaffold an extra one (Postgres allows only a single
|
||||
# default partition per parent). We only need the scaffolding
|
||||
# default partition - dropped again once the data copy is done -
|
||||
# when the source has no default partition, to prevent the
|
||||
# row-copy INSERT from failing on rows that don't match any of the
|
||||
# real partitions.
|
||||
#
|
||||
# Hash-partitioned tables cannot have a default partition at all,
|
||||
# so none is scaffolded for them. With the full set of remainders
|
||||
# every row hashes to some partition anyway. If the source leaves
|
||||
# remainders uncovered, the row-copy INSERT fails before anything
|
||||
# has been dropped, and as the generated script runs in a single
|
||||
# transaction the whole rebuild is rolled back, so no data is lost.
|
||||
source_has_default_partition = any(
|
||||
partition.get('is_default') for partition in source_partitions)
|
||||
target_data['create_scaffolding_default_partition'] = \
|
||||
not source_has_default_partition and \
|
||||
source_data.get('partition_type') != 'hash'
|
||||
|
||||
# For PG/EPAS 11 and above when we copy the data from original
|
||||
# table to temporary table for schema diff, we will have to create
|
||||
# a default partition to prevent the data loss.
|
||||
# a default partition to prevent the data loss. Derive its name
|
||||
# from the already-randomised temporary table name (rather than
|
||||
# the original table's name) so it cannot collide with an
|
||||
# existing relation.
|
||||
target_data['default_partition_name'] = \
|
||||
target_data['orig_name'] + '_default'
|
||||
target_data['name'] + '_default'
|
||||
|
||||
# Copy the partition scheme from source to target.
|
||||
if 'partition_scheme' in source_data:
|
||||
|
||||
+14
@@ -322,6 +322,20 @@ class SchemaDiffTableCompare(SchemaDiffObjectCompare):
|
||||
pk_diff = self.table_constraint_comp(source, target)
|
||||
diff_dict.update(pk_diff)
|
||||
|
||||
# When both the source and target tables are partitioned, the
|
||||
# 'partition' submodule branch below rebuilds the whole table
|
||||
# (and every one of its partitions) from scratch, handling any
|
||||
# added, removed or bound-changed partition itself. The generic
|
||||
# partition add/remove handling in get_sql_from_table_diff (via
|
||||
# _check_for_partitions_in_sql) must not also run in that case:
|
||||
# it would detach/recreate the same partitions using their real
|
||||
# names ahead of the rebuild script, so the rebuild's row-copy
|
||||
# step (which reads from the original table) would miss rows
|
||||
# already detached out from under it, silently losing data.
|
||||
if 'partitions' in diff_dict and \
|
||||
source.get('is_partitioned') and target.get('is_partitioned'):
|
||||
del diff_dict['partitions']
|
||||
|
||||
# Get the difference DDL/DML statements for table
|
||||
target_params['diff_data'] = diff_dict
|
||||
diff = self.get_sql_from_table_diff(**target_params)
|
||||
|
||||
+21
-3
@@ -1,15 +1,33 @@
|
||||
CREATE TABLE IF NOT EXISTS {{conn|qtIdent(data.schema, data.name)}} (
|
||||
LIKE {{conn|qtIdent(data.schema, data.orig_name)}} INCLUDING ALL
|
||||
) PARTITION BY {{ data.partition_scheme }};
|
||||
{{partition_sql}}{{partition_data.default_partition_header}}
|
||||
{{partition_sql}}{% if data.create_scaffolding_default_partition %}{{partition_data.default_partition_header}}
|
||||
CREATE TABLE IF NOT EXISTS {{conn|qtIdent(data.schema, data.default_partition_name)}} PARTITION OF {{conn|qtIdent(data.schema, data.name)}} DEFAULT;
|
||||
|
||||
{% endif %}
|
||||
INSERT INTO {{conn|qtIdent(data.schema, data.name)}}(
|
||||
{% if data.columns and data.columns|length > 0 %}
|
||||
{% for c in data.columns %} {{c.name}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %})
|
||||
SELECT {% if data.columns and data.columns|length > 0 %}{% for c in data.columns %}{{c.name}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}
|
||||
FROM {{conn|qtIdent(data.schema, data.orig_name)}};
|
||||
|
||||
{% if data.create_scaffolding_default_partition %}
|
||||
-- The source table has no default partition of its own, so the
|
||||
-- scaffolding default partition created above (purely to stop the row
|
||||
-- copy above failing on unmatched rows) is dropped, but only if it is
|
||||
-- still empty. If any rows were routed into it (i.e. rows that don't
|
||||
-- fall within the bounds of any other partition), it is left in place
|
||||
-- so that data is not lost; it becomes the default partition of the
|
||||
-- rebuilt table, but keeps its temporary name, so you may want to
|
||||
-- rename it, or move its rows into a suitable partition and drop it.
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM {{conn|qtIdent(data.schema, data.default_partition_name)}}
|
||||
) THEN
|
||||
DROP TABLE {{conn|qtIdent(data.schema, data.default_partition_name)}};
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
{% endif %}
|
||||
{% if partition_data.partitions and partition_data.partitions|length > 0 %}
|
||||
{% for part in partition_data.partitions %}
|
||||
DROP TABLE IF EXISTS {{conn|qtIdent(data.schema, part.partition_name)}};
|
||||
|
||||
+21
-3
@@ -1,15 +1,33 @@
|
||||
CREATE TABLE {{conn|qtIdent(data.schema, data.name)}} (
|
||||
LIKE {{conn|qtIdent(data.schema, data.orig_name)}} INCLUDING ALL
|
||||
) PARTITION BY {{ data.partition_scheme }};
|
||||
{{partition_sql}}{{partition_data.default_partition_header}}
|
||||
{{partition_sql}}{% if data.create_scaffolding_default_partition %}{{partition_data.default_partition_header}}
|
||||
CREATE TABLE IF NOT EXISTS {{conn|qtIdent(data.schema, data.default_partition_name)}} PARTITION OF {{conn|qtIdent(data.schema, data.name)}} DEFAULT;
|
||||
|
||||
{% endif %}
|
||||
INSERT INTO {{conn|qtIdent(data.schema, data.name)}}(
|
||||
{% if data.columns and data.columns|length > 0 %}
|
||||
{% for c in data.columns %} {{c.name}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %})
|
||||
SELECT {% if data.columns and data.columns|length > 0 %}{% for c in data.columns %}{{c.name}}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}
|
||||
FROM {{conn|qtIdent(data.schema, data.orig_name)}};
|
||||
|
||||
{% if data.create_scaffolding_default_partition %}
|
||||
-- The source table has no default partition of its own, so the
|
||||
-- scaffolding default partition created above (purely to stop the row
|
||||
-- copy above failing on unmatched rows) is dropped, but only if it is
|
||||
-- still empty. If any rows were routed into it (i.e. rows that don't
|
||||
-- fall within the bounds of any other partition), it is left in place
|
||||
-- so that data is not lost; it becomes the default partition of the
|
||||
-- rebuilt table, but keeps its temporary name, so you may want to
|
||||
-- rename it, or move its rows into a suitable partition and drop it.
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM {{conn|qtIdent(data.schema, data.default_partition_name)}}
|
||||
) THEN
|
||||
DROP TABLE {{conn|qtIdent(data.schema, data.default_partition_name)}};
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
{% endif %}
|
||||
{% if partition_data.partitions and partition_data.partitions|length > 0 %}
|
||||
{% for part in partition_data.partitions %}
|
||||
DROP TABLE IF EXISTS {{conn|qtIdent(data.schema, part.partition_name)}};
|
||||
|
||||
@@ -33,11 +33,7 @@ class SchemaDiffTestCase(BaseSocketTestGenerator):
|
||||
# with the issue that covers it. The test fails on anything outside
|
||||
# this list, and also fails when something on it starts working, so
|
||||
# that the list cannot quietly rot.
|
||||
KNOWN_DIFFERENCES = {
|
||||
# Rebuilding a partitioned table leaves its scaffolding default
|
||||
# partition behind.
|
||||
'table table_for_partition_1': 10301,
|
||||
}
|
||||
KNOWN_DIFFERENCES = {}
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
||||
@@ -0,0 +1,439 @@
|
||||
##########################################################################
|
||||
#
|
||||
# pgAdmin 4 - PostgreSQL Tools
|
||||
#
|
||||
# Copyright (C) 2013 - 2026, The pgAdmin Development Team
|
||||
# This software is released under the PostgreSQL Licence
|
||||
#
|
||||
##########################################################################
|
||||
|
||||
"""Schema Diff tests for partitioned table rebuilds (#10301).
|
||||
|
||||
When Schema Diff has to rebuild a partitioned table (e.g. because a
|
||||
partition's bounds changed) it creates a temporary partitioned table, adds
|
||||
a scaffolding DEFAULT partition to it so the row-copy INSERT does not fail
|
||||
on rows that match none of the real partitions, copies the rows across and
|
||||
then renames everything into place. Two things must hold once that is
|
||||
done:
|
||||
|
||||
* If the source table has no default partition of its own, the scaffolding
|
||||
one must be dropped again, otherwise the rebuilt target ends up with an
|
||||
extra default partition the source never had.
|
||||
* If the source table genuinely has a default partition, only one DEFAULT
|
||||
partition may exist on the temporary table (Postgres allows a single
|
||||
one), and it must be the real one, not the scaffolding one.
|
||||
* If the target table already holds rows that fall outside every one of
|
||||
the rebuilt partitions' bounds (so they land in the scaffolding DEFAULT
|
||||
partition during the copy), those rows must not be silently discarded
|
||||
when the scaffolding partition would otherwise be dropped.
|
||||
* Hash-partitioned tables cannot have a DEFAULT partition at all, so no
|
||||
scaffolding one may be created for them, and the rows must still be
|
||||
copied into the rebuilt table.
|
||||
"""
|
||||
|
||||
import json
|
||||
import secrets
|
||||
import uuid
|
||||
|
||||
from pgadmin.utils.route import BaseSocketTestGenerator
|
||||
from regression import parent_node_dict
|
||||
from regression.python_test_utils import test_utils as utils
|
||||
|
||||
SCHEMA_NAME = 'test_partition_default_diff'
|
||||
|
||||
DDL_SOURCE = """
|
||||
CREATE SCHEMA {0};
|
||||
|
||||
CREATE TABLE {0}.part_no_default (
|
||||
col1 integer NOT NULL
|
||||
) PARTITION BY RANGE (col1);
|
||||
|
||||
CREATE TABLE {0}.part_no_default_p1 PARTITION OF {0}.part_no_default
|
||||
FOR VALUES FROM (1) TO (10);
|
||||
CREATE TABLE {0}.part_no_default_p2 PARTITION OF {0}.part_no_default
|
||||
FOR VALUES FROM (10) TO (20);
|
||||
|
||||
CREATE TABLE {0}.part_with_default (
|
||||
col1 integer NOT NULL
|
||||
) PARTITION BY RANGE (col1);
|
||||
|
||||
CREATE TABLE {0}.part_with_default_p1 PARTITION OF {0}.part_with_default
|
||||
FOR VALUES FROM (1) TO (10);
|
||||
CREATE TABLE {0}.part_with_default_def PARTITION OF {0}.part_with_default
|
||||
DEFAULT;
|
||||
|
||||
CREATE TABLE {0}.part_stray_rows (
|
||||
col1 integer NOT NULL
|
||||
) PARTITION BY RANGE (col1);
|
||||
|
||||
CREATE TABLE {0}.part_stray_rows_p1 PARTITION OF {0}.part_stray_rows
|
||||
FOR VALUES FROM (1) TO (10);
|
||||
|
||||
CREATE TABLE {0}.part_hash (
|
||||
col1 integer NOT NULL,
|
||||
col2 text
|
||||
) PARTITION BY HASH (col1);
|
||||
|
||||
CREATE TABLE {0}.part_hash_p0 PARTITION OF {0}.part_hash
|
||||
FOR VALUES WITH (MODULUS 2, REMAINDER 0);
|
||||
CREATE TABLE {0}.part_hash_p1 PARTITION OF {0}.part_hash
|
||||
FOR VALUES WITH (MODULUS 2, REMAINDER 1);
|
||||
"""
|
||||
|
||||
# The target starts out with different partition bounds so that Schema Diff
|
||||
# has to rebuild both tables; the default-partition shape of each table
|
||||
# matches its source counterpart's *before* the fix, i.e. still wrong,
|
||||
# forcing Schema Diff's generated DDL to correct it.
|
||||
DDL_TARGET = """
|
||||
CREATE SCHEMA {0};
|
||||
|
||||
CREATE TABLE {0}.part_no_default (
|
||||
col1 integer NOT NULL
|
||||
) PARTITION BY RANGE (col1);
|
||||
|
||||
CREATE TABLE {0}.part_no_default_p1 PARTITION OF {0}.part_no_default
|
||||
FOR VALUES FROM (1) TO (5);
|
||||
|
||||
INSERT INTO {0}.part_no_default_p1 VALUES (2), (3);
|
||||
|
||||
CREATE TABLE {0}.part_with_default (
|
||||
col1 integer NOT NULL
|
||||
) PARTITION BY RANGE (col1);
|
||||
|
||||
CREATE TABLE {0}.part_with_default_p1 PARTITION OF {0}.part_with_default
|
||||
FOR VALUES FROM (1) TO (5);
|
||||
CREATE TABLE {0}.part_with_default_def PARTITION OF {0}.part_with_default
|
||||
DEFAULT;
|
||||
|
||||
-- col1=2 fits the regular partition both before and after the rebuild;
|
||||
-- col1=50 fits neither the old nor the new bounds of the regular
|
||||
-- partition, so it must land (and stay) in the genuine DEFAULT partition
|
||||
-- across the rebuild.
|
||||
INSERT INTO {0}.part_with_default VALUES (2), (50);
|
||||
|
||||
CREATE TABLE {0}.part_stray_rows (
|
||||
col1 integer NOT NULL
|
||||
) PARTITION BY RANGE (col1);
|
||||
|
||||
CREATE TABLE {0}.part_stray_rows_p1 PARTITION OF {0}.part_stray_rows
|
||||
FOR VALUES FROM (1) TO (5);
|
||||
CREATE TABLE {0}.part_stray_rows_def PARTITION OF {0}.part_stray_rows
|
||||
DEFAULT;
|
||||
|
||||
-- col1=2 fits the rebuilt regular partition's new bounds; col1=500 fits
|
||||
-- neither the old nor the new bounds of any regular partition, so it is
|
||||
-- only reachable via a DEFAULT partition. The source table has no
|
||||
-- DEFAULT partition of its own, so this row can only survive the
|
||||
-- rebuild if the scaffolding DEFAULT partition is kept instead of being
|
||||
-- unconditionally dropped once it holds data.
|
||||
INSERT INTO {0}.part_stray_rows VALUES (2), (500);
|
||||
|
||||
-- The source adds col2, which makes Schema Diff rebuild the table. Hash
|
||||
-- partitions cannot have a DEFAULT sibling, so the rebuild must not try
|
||||
-- to scaffold one, and the rows spread across both partitions must be
|
||||
-- copied into the rebuilt table.
|
||||
CREATE TABLE {0}.part_hash (
|
||||
col1 integer NOT NULL
|
||||
) PARTITION BY HASH (col1);
|
||||
|
||||
CREATE TABLE {0}.part_hash_p0 PARTITION OF {0}.part_hash
|
||||
FOR VALUES WITH (MODULUS 2, REMAINDER 0);
|
||||
CREATE TABLE {0}.part_hash_p1 PARTITION OF {0}.part_hash
|
||||
FOR VALUES WITH (MODULUS 2, REMAINDER 1);
|
||||
|
||||
INSERT INTO {0}.part_hash SELECT generate_series(1, 10);
|
||||
"""
|
||||
|
||||
|
||||
class SchemaDiffPartitionDefaultTestCase(BaseSocketTestGenerator):
|
||||
""" This class will test Schema Diff against partitioned table
|
||||
rebuilds involving default partitions. """
|
||||
scenarios = [
|
||||
('Schema diff comparison of partition rebuild default partitions',
|
||||
dict())
|
||||
]
|
||||
SOCKET_NAMESPACE = '/schema_diff'
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.src_database = "db_part_diff_src_%s" % str(uuid.uuid4())[1:8]
|
||||
self.tar_database = "db_part_diff_tar_%s" % str(uuid.uuid4())[1:8]
|
||||
|
||||
self.src_db_id = utils.create_database(self.server, self.src_database)
|
||||
self.tar_db_id = utils.create_database(self.server, self.tar_database)
|
||||
|
||||
self.server = parent_node_dict["server"][-1]["server"]
|
||||
self.server_id = parent_node_dict["server"][-1]["server_id"]
|
||||
|
||||
self.execute_sql(self.src_database, DDL_SOURCE.format(SCHEMA_NAME))
|
||||
self.execute_sql(self.tar_database, DDL_TARGET.format(SCHEMA_NAME))
|
||||
|
||||
def execute_sql(self, db_name, sql):
|
||||
"""
|
||||
Run a statement batch against one of the test databases.
|
||||
|
||||
:param db_name: Database to run against
|
||||
:param sql: SQL to execute
|
||||
"""
|
||||
connection = utils.get_db_connection(db_name,
|
||||
self.server['username'],
|
||||
self.server['db_password'],
|
||||
self.server['host'],
|
||||
self.server['port'],
|
||||
self.server['sslmode'])
|
||||
old_isolation_level = connection.isolation_level
|
||||
utils.set_isolation_level(connection, 0)
|
||||
pg_cursor = connection.cursor()
|
||||
pg_cursor.execute(sql)
|
||||
utils.set_isolation_level(connection, old_isolation_level)
|
||||
connection.commit()
|
||||
connection.close()
|
||||
|
||||
def fetch_scalar(self, db_name, sql):
|
||||
"""
|
||||
Run a query against one of the test databases and return the
|
||||
first column of the first row.
|
||||
|
||||
:param db_name: Database to run against
|
||||
:param sql: SQL to execute
|
||||
"""
|
||||
connection = utils.get_db_connection(db_name,
|
||||
self.server['username'],
|
||||
self.server['db_password'],
|
||||
self.server['host'],
|
||||
self.server['port'],
|
||||
self.server['sslmode'])
|
||||
pg_cursor = connection.cursor()
|
||||
pg_cursor.execute(sql)
|
||||
result = pg_cursor.fetchone()[0]
|
||||
connection.close()
|
||||
return int(result)
|
||||
|
||||
def count_default_partitions(self, db_name, table_name):
|
||||
"""
|
||||
Count how many DEFAULT partitions the given partitioned table has
|
||||
in the given database.
|
||||
"""
|
||||
return self.fetch_scalar(
|
||||
db_name,
|
||||
"SELECT count(*) FROM pg_inherits i "
|
||||
"JOIN pg_class c ON c.oid = i.inhrelid "
|
||||
"JOIN pg_class p ON p.oid = i.inhparent "
|
||||
"WHERE p.relname = '{0}' "
|
||||
"AND p.relnamespace = '{1}'::regnamespace "
|
||||
"AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'".format(
|
||||
table_name, SCHEMA_NAME))
|
||||
|
||||
def count_rows(self, db_name, table_name):
|
||||
"""
|
||||
Count how many rows the given table holds in the given database.
|
||||
"""
|
||||
return self.fetch_scalar(
|
||||
db_name,
|
||||
"SELECT count(*) FROM {0}.{1}".format(SCHEMA_NAME, table_name))
|
||||
|
||||
def count_partitions(self, db_name, table_name):
|
||||
"""
|
||||
Count how many partitions the given partitioned table has in the
|
||||
given database.
|
||||
"""
|
||||
return self.fetch_scalar(
|
||||
db_name,
|
||||
"SELECT count(*) FROM pg_inherits i "
|
||||
"JOIN pg_class p ON p.oid = i.inhparent "
|
||||
"WHERE p.relname = '{0}' "
|
||||
"AND p.relnamespace = '{1}'::regnamespace".format(
|
||||
table_name, SCHEMA_NAME))
|
||||
|
||||
def compare(self):
|
||||
"""
|
||||
Compare the two test databases and return the result.
|
||||
|
||||
:return: List of compared objects
|
||||
"""
|
||||
data = {
|
||||
'trans_id': self.trans_id,
|
||||
'source_sid': self.server_id,
|
||||
'source_did': self.src_db_id,
|
||||
'target_sid': self.server_id,
|
||||
'target_did': self.tar_db_id,
|
||||
'ignore_owner': 0,
|
||||
'ignore_whitespaces': 0,
|
||||
'ignore_tablespace': 0,
|
||||
'ignore_grants': 0
|
||||
}
|
||||
self.socket_client.emit('compare_database', data,
|
||||
namespace=self.SOCKET_NAMESPACE)
|
||||
received = self.socket_client.get_received(self.SOCKET_NAMESPACE)
|
||||
response_data = received[-1]['args'][0]
|
||||
self.assertEqual(received[-1]['name'], "compare_database_success",
|
||||
response_data)
|
||||
return response_data
|
||||
|
||||
def find_object(self, response_data, node_type, title):
|
||||
"""
|
||||
Pick a single compared object out of the comparison result.
|
||||
|
||||
:param response_data: Result of compare()
|
||||
:param node_type: Node type, e.g. 'table'
|
||||
:param title: Object name
|
||||
:return: The compared object
|
||||
"""
|
||||
for diff in response_data:
|
||||
if diff.get('type') == node_type and diff.get('title') == title:
|
||||
return diff
|
||||
|
||||
self.fail('{0} {1} was not compared'.format(node_type, title))
|
||||
|
||||
def runTest(self):
|
||||
""" This function will test Schema Diff for partition rebuilds
|
||||
that involve default partitions. """
|
||||
self.trans_id = str(secrets.choice(range(1, 99999)))
|
||||
response = self.tester.get(
|
||||
'schema_diff/initialize/{}'.format(self.trans_id))
|
||||
self.assertEqual(response.status_code, 200)
|
||||
|
||||
received = self.socket_client.get_received(self.SOCKET_NAMESPACE)
|
||||
self.assertEqual(received[0]['name'], 'connected')
|
||||
|
||||
self.tester.post(
|
||||
'schema_diff/server/connect/{}'.format(self.server_id),
|
||||
data=json.dumps({'password': self.server['db_password']}),
|
||||
content_type='html/json')
|
||||
self.tester.post('schema_diff/database/connect/{0}/{1}'.format(
|
||||
self.server_id, self.src_db_id))
|
||||
self.tester.post('schema_diff/database/connect/{0}/{1}'.format(
|
||||
self.server_id, self.tar_db_id))
|
||||
|
||||
response_data = self.compare()
|
||||
|
||||
# --- Source has no default partition of its own. ---
|
||||
no_default = self.find_object(response_data, 'table',
|
||||
'part_no_default')
|
||||
self.assertEqual(no_default['status'], 'Different')
|
||||
diff_ddl = no_default['diff_ddl']
|
||||
|
||||
# Applying the diff must succeed and leave no default partition
|
||||
# behind, since the source table doesn't have one.
|
||||
self.execute_sql(self.tar_database, diff_ddl)
|
||||
self.assertEqual(
|
||||
self.count_default_partitions(self.tar_database,
|
||||
'part_no_default'), 0,
|
||||
'Schema Diff left a scaffolding default partition behind on '
|
||||
'a table whose source has no default partition: '
|
||||
'{0}'.format(diff_ddl))
|
||||
self.assertEqual(
|
||||
self.count_partitions(self.tar_database, 'part_no_default'), 2)
|
||||
# The rows that were copied via the scaffolding default partition
|
||||
# must still be there once it is dropped.
|
||||
self.assertEqual(
|
||||
self.count_rows(self.tar_database, 'part_no_default'), 2,
|
||||
'Schema Diff lost rows while rebuilding a table whose source '
|
||||
'has no default partition: {0}'.format(diff_ddl))
|
||||
|
||||
# --- Source has a genuine default partition. ---
|
||||
with_default = self.find_object(response_data, 'table',
|
||||
'part_with_default')
|
||||
self.assertEqual(with_default['status'], 'Different')
|
||||
diff_ddl = with_default['diff_ddl']
|
||||
|
||||
# Applying the diff must succeed (it must not try to create two
|
||||
# DEFAULT partitions on the temporary table) and the real default
|
||||
# partition must survive.
|
||||
self.execute_sql(self.tar_database, diff_ddl)
|
||||
self.assertEqual(
|
||||
self.count_default_partitions(self.tar_database,
|
||||
'part_with_default'), 1,
|
||||
'Schema Diff did not preserve the source table\'s genuine '
|
||||
'default partition: {0}'.format(diff_ddl))
|
||||
self.assertEqual(
|
||||
self.count_partitions(self.tar_database, 'part_with_default'),
|
||||
2)
|
||||
# The row that fits the regular partition and the row that only
|
||||
# ever fit the DEFAULT partition must both survive the rebuild.
|
||||
self.assertEqual(
|
||||
self.count_rows(self.tar_database, 'part_with_default'), 2,
|
||||
'Schema Diff lost rows while rebuilding a table whose source '
|
||||
'has a genuine default partition: {0}'.format(diff_ddl))
|
||||
|
||||
# --- Target has rows outside every rebuilt partition's bounds. ---
|
||||
# The source table has no default partition of its own, so a
|
||||
# scaffolding one is created purely to let the row-copy INSERT
|
||||
# succeed. col1=500 in the target doesn't fit any real partition
|
||||
# in the rebuilt scheme, so it can only be copied via that
|
||||
# scaffolding partition; it must not be lost when the scaffolding
|
||||
# partition is cleaned up.
|
||||
stray_rows = self.find_object(response_data, 'table',
|
||||
'part_stray_rows')
|
||||
self.assertEqual(stray_rows['status'], 'Different')
|
||||
diff_ddl = stray_rows['diff_ddl']
|
||||
|
||||
self.execute_sql(self.tar_database, diff_ddl)
|
||||
self.assertEqual(
|
||||
self.count_rows(self.tar_database, 'part_stray_rows'), 2,
|
||||
'Schema Diff silently dropped a row that fell outside every '
|
||||
'rebuilt partition\'s bounds: {0}'.format(diff_ddl))
|
||||
self.assertEqual(
|
||||
self.fetch_scalar(
|
||||
self.tar_database,
|
||||
"SELECT count(*) FROM {0}.part_stray_rows "
|
||||
"WHERE col1 = 500".format(SCHEMA_NAME)), 1,
|
||||
'Schema Diff dropped the out-of-bounds row instead of routing '
|
||||
'it to a durable partition: {0}'.format(diff_ddl))
|
||||
self.assertEqual(
|
||||
self.count_default_partitions(self.tar_database,
|
||||
'part_stray_rows'), 1,
|
||||
'Schema Diff dropped the scaffolding default partition even '
|
||||
'though it still held an out-of-bounds row: {0}'.format(
|
||||
diff_ddl))
|
||||
self.assertEqual(
|
||||
self.count_partitions(self.tar_database, 'part_stray_rows'), 2)
|
||||
|
||||
# --- Hash-partitioned table whose source adds a column. ---
|
||||
part_hash = self.find_object(response_data, 'table', 'part_hash')
|
||||
self.assertEqual(part_hash['status'], 'Different')
|
||||
diff_ddl = part_hash['diff_ddl']
|
||||
self.assertNotIn(
|
||||
' DEFAULT;', diff_ddl,
|
||||
'Schema Diff tried to create a scaffolding default partition '
|
||||
'on a hash-partitioned table: {0}'.format(diff_ddl))
|
||||
|
||||
self.execute_sql(self.tar_database, diff_ddl)
|
||||
self.assertEqual(
|
||||
self.count_partitions(self.tar_database, 'part_hash'), 2)
|
||||
self.assertEqual(
|
||||
self.count_rows(self.tar_database, 'part_hash'), 10,
|
||||
'Schema Diff lost rows while rebuilding a hash-partitioned '
|
||||
'table: {0}'.format(diff_ddl))
|
||||
|
||||
# Re-comparing must now report the bounds-only tables and the
|
||||
# hash-partitioned table as identical.
|
||||
response_data = self.compare()
|
||||
self.assertEqual(
|
||||
self.find_object(response_data, 'table',
|
||||
'part_no_default')['status'], 'Identical')
|
||||
self.assertEqual(
|
||||
self.find_object(response_data, 'table',
|
||||
'part_with_default')['status'], 'Identical')
|
||||
self.assertEqual(
|
||||
self.find_object(response_data, 'table',
|
||||
'part_hash')['status'], 'Identical')
|
||||
# part_stray_rows legitimately still differs from its source: the
|
||||
# target kept a default partition (holding the recovered
|
||||
# out-of-bounds row) that the source doesn't have. Preserving the
|
||||
# data takes priority over reporting a false "Identical".
|
||||
self.assertEqual(
|
||||
self.find_object(response_data, 'table',
|
||||
'part_stray_rows')['status'], 'Different')
|
||||
|
||||
def tearDown(self):
|
||||
"""This function drops the added databases"""
|
||||
super().tearDown()
|
||||
for db_name in (self.src_database, self.tar_database):
|
||||
connection = utils.get_db_connection(self.server['db'],
|
||||
self.server['username'],
|
||||
self.server['db_password'],
|
||||
self.server['host'],
|
||||
self.server['port'],
|
||||
self.server['sslmode'])
|
||||
utils.drop_database(connection, db_name)
|
||||
Reference in new issue
Block a user