From 5d45aa81e0dc64684cb6799e6bda057a7829bef4 Mon Sep 17 00:00:00 2001 From: Jesse Hallam Date: Tue, 15 Oct 2019 15:47:09 -0300 Subject: [PATCH] MM-16888: fix missing indexes (#12746) * MM-16888: fix missing indexes As part of https://mattermost.atlassian.net/browse/MM-16888, we discovered and fixed a number of column and index mismatches between the canonical (i.e. created from scratch) and migrated schemas (i.e migrated from 5.0 through 5.16). Unfortunately, the migration to fix same was added to `UpgradeDatabaseToVersion514` but never cherry picked to the pending v5.14 release at the time. Customers who upgraded to v5.14 or v5.15 and then get this code as part of v5.16 will never run that migration. Copy it to the UpgradeDatabaseToVersion516 accordingly. * avoid fixing ChannelMembers.SchemeGuest on MySQL * synchronize .circleci/config.yml with scripts/mysql-migration-test.sh * fix circleci invocation * additional logging on diff * update build/Jenkinsfile.pr too! --- .circleci/config.yml | 9 +++++++-- build/Jenkinsfile.pr | 4 ++++ scripts/mysql-migration-test.sh | 11 ++++++++++- scripts/psql-migration-test.sh | 7 ++++++- store/sqlstore/upgrade.go | 27 ++++++++++++++++----------- 5 files changed, 43 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9ce3c3b4ee..57a72507b2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -205,7 +205,7 @@ jobs: docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test' echo "Generating diff" - diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (cat diff.txt && exit 1) + diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) no_output_timeout: 1h - run: name: MySQL schema migration validation @@ -234,6 +234,11 @@ jobs: -w /go/src/github.com/mattermost/mattermost-server \ mattermost/mattermost-build-server:feb-28-2019 \ bash -c 'ulimit -n 8096; make ARGS="version" run-cli && make MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" ARGS="version" run-cli' + + echo "Ignoring known MySQL mismatch: ChannelMembers.SchemeGuest" + docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;" + docker-compose --no-ansi exec -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;" + echo "Generating dump" docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql @@ -242,7 +247,7 @@ jobs: docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest' echo "Generating diff" - diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (cat diff.txt && exit 1) + diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) no_output_timeout: 1h upload-s3-sha: docker: diff --git a/build/Jenkinsfile.pr b/build/Jenkinsfile.pr index bf8ad7e5d2..21af2e7d34 100644 --- a/build/Jenkinsfile.pr +++ b/build/Jenkinsfile.pr @@ -353,6 +353,10 @@ pipeline { dir('src/github.com/mattermost/mattermost-server') { ansiColor('xterm') { sh """ + echo "Ignoring known MySQL mismatch: ChannelMembers.SchemeGuest" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;" + echo "Generating dump" /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql diff --git a/scripts/mysql-migration-test.sh b/scripts/mysql-migration-test.sh index 5da4cdde5b..1c8f9d3508 100755 --- a/scripts/mysql-migration-test.sh +++ b/scripts/mysql-migration-test.sh @@ -22,6 +22,10 @@ make ARGS="config set SqlSettings.DataSource 'mmuser:mostest@tcp(localhost:3306) echo "Setting up fresh db" make ARGS="version --config $TMPDIR/config.json" run-cli +echo "Ignoring known MySQL mismatch: ChannelMembers.SchemeGuest" +docker exec mattermost-mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;" +docker exec mattermost-mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;" + echo "Generating dump" docker exec mattermost-mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > $DUMPDIR/migrated.sql docker exec mattermost-mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > $DUMPDIR/latest.sql @@ -33,7 +37,12 @@ echo "Generating diff" diff $DUMPDIR/migrated.sql $DUMPDIR/latest.sql > $DUMPDIR/diff.txt diffErrorCode=$? -if [ $diffErrorCode -eq 0 ]; then echo "Both schemas are same";else cat $DUMPDIR/diff.txt; fi +if [ $diffErrorCode -eq 0 ]; then + echo "Both schemas are same" +else + echo "Schema mismatch" + cat $DUMPDIR/diff.txt +fi rm -rf $TMPDIR $DUMPDIR exit $diffErrorCode diff --git a/scripts/psql-migration-test.sh b/scripts/psql-migration-test.sh index 57997bcae1..a3861f4b7d 100755 --- a/scripts/psql-migration-test.sh +++ b/scripts/psql-migration-test.sh @@ -33,7 +33,12 @@ echo "Generating diff" diff $DUMPDIR/migrated.sql $DUMPDIR/latest.sql > $DUMPDIR/diff.txt diffErrorCode=$? -if [ $diffErrorCode -eq 0 ]; then echo "Both schemas are same";else cat $DUMPDIR/diff.txt; fi +if [ $diffErrorCode -eq 0 ]; then + echo "Both schemas are same" +else + echo "Schema mismatch" + cat $DUMPDIR/diff.txt +fi rm -rf $TMPDIR $DUMPDIR exit $diffErrorCode diff --git a/store/sqlstore/upgrade.go b/store/sqlstore/upgrade.go index d645282b95..4b216a4a7a 100644 --- a/store/sqlstore/upgrade.go +++ b/store/sqlstore/upgrade.go @@ -713,17 +713,6 @@ func UpgradeDatabaseToVersion513(sqlStore SqlStore) { func UpgradeDatabaseToVersion514(sqlStore SqlStore) { if shouldPerformUpgrade(sqlStore, VERSION_5_13_0, VERSION_5_14_0) { - sqlStore.AlterColumnTypeIfExists("TeamMembers", "SchemeGuest", "tinyint(4)", "boolean") - sqlStore.AlterColumnTypeIfExists("ChannelMembers", "SchemeGuest", "tinyint(4)", "boolean") - sqlStore.AlterColumnTypeIfExists("Schemes", "DefaultTeamGuestRole", "varchar(64)", "VARCHAR(64)") - sqlStore.AlterColumnTypeIfExists("Schemes", "DefaultChannelGuestRole", "varchar(64)", "VARCHAR(64)") - sqlStore.AlterColumnTypeIfExists("Teams", "AllowedDomains", "text", "VARCHAR(1000)") - sqlStore.AlterColumnTypeIfExists("Channels", "GroupConstrained", "tinyint(1)", "boolean") - sqlStore.AlterColumnTypeIfExists("Teams", "GroupConstrained", "tinyint(1)", "boolean") - - sqlStore.CreateIndexIfNotExists("idx_groupteams_teamid", "GroupTeams", "TeamId") - sqlStore.CreateIndexIfNotExists("idx_groupchannels_channelid", "GroupChannels", "ChannelId") - saveSchemaVersion(sqlStore, VERSION_5_14_0) } } @@ -742,5 +731,21 @@ func UpgradeDatabaseToVersion516(sqlStore SqlStore) { sqlStore.GetMaster().Exec("ALTER TABLE Tokens MODIFY Extra text") } saveSchemaVersion(sqlStore, VERSION_5_16_0) + + // Fix mismatches between the canonical and migrated schemas. + sqlStore.AlterColumnTypeIfExists("TeamMembers", "SchemeGuest", "tinyint(4)", "boolean") + sqlStore.AlterColumnTypeIfExists("Schemes", "DefaultTeamGuestRole", "varchar(64)", "VARCHAR(64)") + sqlStore.AlterColumnTypeIfExists("Schemes", "DefaultChannelGuestRole", "varchar(64)", "VARCHAR(64)") + sqlStore.AlterColumnTypeIfExists("Teams", "AllowedDomains", "text", "VARCHAR(1000)") + sqlStore.AlterColumnTypeIfExists("Channels", "GroupConstrained", "tinyint(1)", "boolean") + sqlStore.AlterColumnTypeIfExists("Teams", "GroupConstrained", "tinyint(1)", "boolean") + + // One known mismatch remains: ChannelMembers.SchemeGuest. The requisite migration + // is left here for posterity, but we're avoiding fix this given the corresponding + // table rewrite in most MySQL and Postgres instances. + // sqlStore.AlterColumnTypeIfExists("ChannelMembers", "SchemeGuest", "tinyint(4)", "boolean") + + sqlStore.CreateIndexIfNotExists("idx_groupteams_teamid", "GroupTeams", "TeamId") + sqlStore.CreateIndexIfNotExists("idx_groupchannels_channelid", "GroupChannels", "ChannelId") } }