From 24d4e50343ef0fd9dc6209064637773a73724274 Mon Sep 17 00:00:00 2001 From: Dan Cech Date: Tue, 28 Mar 2017 08:34:53 -0400 Subject: [PATCH] utf8mb4 encoding (#7959) * use utf8mb4 character set for connections to mysql * use utf8mb4 character set for tables, shorten varchar fields used in unique indexes * migration type to update table character set * update table character sets * set charset for temp_user.status * gofmt --- pkg/services/sqlstore/migrations/alert_mig.go | 17 +++++++++++++- .../sqlstore/migrations/annotation_mig.go | 10 ++++++++ .../sqlstore/migrations/apikey_mig.go | 12 +++++++--- .../sqlstore/migrations/dashboard_mig.go | 15 ++++++++++-- .../migrations/dashboard_snapshot_mig.go | 14 ++++++++--- .../sqlstore/migrations/datasource_mig.go | 18 +++++++++++++-- pkg/services/sqlstore/migrations/org_mig.go | 17 +++++++++++++- .../sqlstore/migrations/playlist_mig.go | 11 +++++++++ .../sqlstore/migrations/plugin_setting.go | 8 ++++++- .../sqlstore/migrations/preferences_mig.go | 5 ++++ pkg/services/sqlstore/migrations/quota_mig.go | 6 ++++- pkg/services/sqlstore/migrations/temp_user.go | 9 ++++++++ pkg/services/sqlstore/migrations/user_mig.go | 19 +++++++++++---- pkg/services/sqlstore/migrator/dialect.go | 7 +++++- pkg/services/sqlstore/migrator/migrations.go | 14 +++++++++++ .../sqlstore/migrator/mysql_dialect.go | 23 ++++++++++++++++++- .../sqlstore/migrator/postgres_dialect.go | 11 +++++++++ pkg/services/sqlstore/sqlstore.go | 2 +- pkg/services/sqlstore/sqlutil/sqlutil.go | 2 +- 19 files changed, 198 insertions(+), 22 deletions(-) diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index b6956be41c7..7b6558656f1 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -50,7 +50,7 @@ func addAlertMigrations(mg *Migrator) { Columns: []*Column{ {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, {Name: "org_id", Type: DB_BigInt, Nullable: false}, - {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false}, {Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "settings", Type: DB_Text, Nullable: false}, {Name: "created", Type: DB_DateTime, Nullable: false}, @@ -67,4 +67,19 @@ func addAlertMigrations(mg *Migrator) { })) mg.AddMigration("add index alert_notification org_id & name", NewAddIndexMigration(alert_notification, alert_notification.Indices[0])) + mg.AddMigration("Update alert table charset", NewTableCharsetMigration("alert", []*Column{ + {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "message", Type: DB_Text, Nullable: false}, + {Name: "state", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "settings", Type: DB_Text, Nullable: false}, + {Name: "severity", Type: DB_Text, Nullable: false}, + {Name: "execution_error", Type: DB_Text, Nullable: false}, + {Name: "eval_data", Type: DB_Text, Nullable: true}, + })) + + mg.AddMigration("Update alert_notification table charset", NewTableCharsetMigration("alert_notification", []*Column{ + {Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "settings", Type: DB_Text, Nullable: false}, + })) } diff --git a/pkg/services/sqlstore/migrations/annotation_mig.go b/pkg/services/sqlstore/migrations/annotation_mig.go index a307100ad3c..4a7206f9d64 100644 --- a/pkg/services/sqlstore/migrations/annotation_mig.go +++ b/pkg/services/sqlstore/migrations/annotation_mig.go @@ -44,4 +44,14 @@ func addAnnotationMig(mg *Migrator) { mg.AddMigration("add index annotation 2 v3", NewAddIndexMigration(table, table.Indices[2])) mg.AddMigration("add index annotation 3 v3", NewAddIndexMigration(table, table.Indices[3])) mg.AddMigration("add index annotation 4 v3", NewAddIndexMigration(table, table.Indices[4])) + + mg.AddMigration("Update annotation table charset", NewTableCharsetMigration("annotation", []*Column{ + {Name: "type", Type: DB_NVarchar, Length: 25, Nullable: false}, + {Name: "title", Type: DB_Text, Nullable: false}, + {Name: "text", Type: DB_Text, Nullable: false}, + {Name: "metric", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "prev_state", Type: DB_NVarchar, Length: 25, Nullable: false}, + {Name: "new_state", Type: DB_NVarchar, Length: 25, Nullable: false}, + {Name: "data", Type: DB_Text, Nullable: false}, + })) } diff --git a/pkg/services/sqlstore/migrations/apikey_mig.go b/pkg/services/sqlstore/migrations/apikey_mig.go index a02e759f551..928f84c4fb0 100644 --- a/pkg/services/sqlstore/migrations/apikey_mig.go +++ b/pkg/services/sqlstore/migrations/apikey_mig.go @@ -8,7 +8,7 @@ func addApiKeyMigrations(mg *Migrator) { Columns: []*Column{ {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, {Name: "account_id", Type: DB_BigInt, Nullable: false}, - {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false}, {Name: "key", Type: DB_Varchar, Length: 64, Nullable: false}, {Name: "role", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "created", Type: DB_DateTime, Nullable: false}, @@ -41,8 +41,8 @@ func addApiKeyMigrations(mg *Migrator) { Columns: []*Column{ {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, {Name: "org_id", Type: DB_BigInt, Nullable: false}, - {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, - {Name: "key", Type: DB_Varchar, Length: 255, Nullable: false}, + {Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "key", Type: DB_Varchar, Length: 190, Nullable: false}, {Name: "role", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "created", Type: DB_DateTime, Nullable: false}, {Name: "updated", Type: DB_DateTime, Nullable: false}, @@ -72,4 +72,10 @@ func addApiKeyMigrations(mg *Migrator) { })) mg.AddMigration("Drop old table api_key_v1", NewDropTableMigration("api_key_v1")) + + mg.AddMigration("Update api_key table charset", NewTableCharsetMigration("api_key", []*Column{ + {Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "key", Type: DB_Varchar, Length: 190, Nullable: false}, + {Name: "role", Type: DB_NVarchar, Length: 255, Nullable: false}, + })) } diff --git a/pkg/services/sqlstore/migrations/dashboard_mig.go b/pkg/services/sqlstore/migrations/dashboard_mig.go index 283501d366f..0ef2f3be54f 100644 --- a/pkg/services/sqlstore/migrations/dashboard_mig.go +++ b/pkg/services/sqlstore/migrations/dashboard_mig.go @@ -8,7 +8,7 @@ func addDashboardMigration(mg *Migrator) { Columns: []*Column{ {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, {Name: "version", Type: DB_Int, Nullable: false}, - {Name: "slug", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "slug", Type: DB_NVarchar, Length: 190, Nullable: false}, {Name: "title", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "data", Type: DB_Text, Nullable: false}, {Name: "account_id", Type: DB_BigInt, Nullable: false}, @@ -56,7 +56,7 @@ func addDashboardMigration(mg *Migrator) { Columns: []*Column{ {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, {Name: "version", Type: DB_Int, Nullable: false}, - {Name: "slug", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "slug", Type: DB_NVarchar, Length: 190, Nullable: false}, {Name: "title", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "data", Type: DB_Text, Nullable: false}, {Name: "org_id", Type: DB_BigInt, Nullable: false}, @@ -125,4 +125,15 @@ func addDashboardMigration(mg *Migrator) { mg.AddMigration("Add index for dashboard_id in dashboard_tag", NewAddIndexMigration(dashboardTagV1, &Index{ Cols: []string{"dashboard_id"}, Type: IndexType, })) + + mg.AddMigration("Update dashboard table charset", NewTableCharsetMigration("dashboard", []*Column{ + {Name: "slug", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "title", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "plugin_id", Type: DB_NVarchar, Nullable: true, Length: 255}, + {Name: "data", Type: DB_MediumText, Nullable: false}, + })) + + mg.AddMigration("Update dashboard_tag table charset", NewTableCharsetMigration("dashboard_tag", []*Column{ + {Name: "term", Type: DB_NVarchar, Length: 50, Nullable: false}, + })) } diff --git a/pkg/services/sqlstore/migrations/dashboard_snapshot_mig.go b/pkg/services/sqlstore/migrations/dashboard_snapshot_mig.go index b08cc451e55..1a72bcba929 100644 --- a/pkg/services/sqlstore/migrations/dashboard_snapshot_mig.go +++ b/pkg/services/sqlstore/migrations/dashboard_snapshot_mig.go @@ -8,7 +8,7 @@ func addDashboardSnapshotMigrations(mg *Migrator) { Columns: []*Column{ {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, - {Name: "key", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "key", Type: DB_NVarchar, Length: 190, Nullable: false}, {Name: "dashboard", Type: DB_Text, Nullable: false}, {Name: "expires", Type: DB_DateTime, Nullable: false}, {Name: "created", Type: DB_DateTime, Nullable: false}, @@ -28,8 +28,8 @@ func addDashboardSnapshotMigrations(mg *Migrator) { Columns: []*Column{ {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, - {Name: "key", Type: DB_NVarchar, Length: 255, Nullable: false}, - {Name: "delete_key", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "key", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "delete_key", Type: DB_NVarchar, Length: 190, Nullable: false}, {Name: "org_id", Type: DB_BigInt, Nullable: false}, {Name: "user_id", Type: DB_BigInt, Nullable: false}, {Name: "external", Type: DB_Bool, Nullable: false}, @@ -54,4 +54,12 @@ func addDashboardSnapshotMigrations(mg *Migrator) { Sqlite("SELECT 0 WHERE 0;"). Postgres("SELECT 0;"). Mysql("ALTER TABLE dashboard_snapshot MODIFY dashboard MEDIUMTEXT;")) + + mg.AddMigration("Update dashboard_snapshot table charset", NewTableCharsetMigration("dashboard_snapshot", []*Column{ + {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "key", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "delete_key", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "external_url", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "dashboard", Type: DB_MediumText, Nullable: false}, + })) } diff --git a/pkg/services/sqlstore/migrations/datasource_mig.go b/pkg/services/sqlstore/migrations/datasource_mig.go index 396f80df99f..c694b5b7cfb 100644 --- a/pkg/services/sqlstore/migrations/datasource_mig.go +++ b/pkg/services/sqlstore/migrations/datasource_mig.go @@ -10,7 +10,7 @@ func addDataSourceMigration(mg *Migrator) { {Name: "account_id", Type: DB_BigInt, Nullable: false}, {Name: "version", Type: DB_Int, Nullable: false}, {Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false}, - {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false}, {Name: "access", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "url", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true}, @@ -49,7 +49,7 @@ func addDataSourceMigration(mg *Migrator) { {Name: "org_id", Type: DB_BigInt, Nullable: false}, {Name: "version", Type: DB_Int, Nullable: false}, {Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false}, - {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false}, {Name: "access", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "url", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true}, @@ -106,4 +106,18 @@ func addDataSourceMigration(mg *Migrator) { mg.AddMigration("Add secure json data column", NewAddColumnMigration(tableV2, &Column{ Name: "secure_json_data", Type: DB_Text, Nullable: true, })) + + mg.AddMigration("Update data_source table charset", NewTableCharsetMigration(tableV2.Name, []*Column{ + {Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "access", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "url", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "user", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "database", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "basic_auth_user", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "basic_auth_password", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "json_data", Type: DB_Text, Nullable: true}, + {Name: "secure_json_data", Type: DB_Text, Nullable: true}, + })) } diff --git a/pkg/services/sqlstore/migrations/org_mig.go b/pkg/services/sqlstore/migrations/org_mig.go index 8e25da76b58..12e0a04256a 100644 --- a/pkg/services/sqlstore/migrations/org_mig.go +++ b/pkg/services/sqlstore/migrations/org_mig.go @@ -8,7 +8,7 @@ func addOrgMigrations(mg *Migrator) { Columns: []*Column{ {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, {Name: "version", Type: DB_Int, Nullable: false}, - {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false}, {Name: "address1", Type: DB_NVarchar, Length: 255, Nullable: true}, {Name: "address2", Type: DB_NVarchar, Length: 255, Nullable: true}, {Name: "city", Type: DB_NVarchar, Length: 255, Nullable: true}, @@ -68,4 +68,19 @@ func addOrgMigrations(mg *Migrator) { mg.AddMigration("Drop old table account", NewDropTableMigration("account")) mg.AddMigration("Drop old table account_user", NewDropTableMigration("account_user")) + + mg.AddMigration("Update org table charset", NewTableCharsetMigration("org", []*Column{ + {Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "address1", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "address2", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "city", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "state", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "zip_code", Type: DB_NVarchar, Length: 50, Nullable: true}, + {Name: "country", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "billing_email", Type: DB_NVarchar, Length: 255, Nullable: true}, + })) + + mg.AddMigration("Update org_user table charset", NewTableCharsetMigration("org_user", []*Column{ + {Name: "role", Type: DB_NVarchar, Length: 20}, + })) } diff --git a/pkg/services/sqlstore/migrations/playlist_mig.go b/pkg/services/sqlstore/migrations/playlist_mig.go index 636fc091006..44ae4b6cb0e 100644 --- a/pkg/services/sqlstore/migrations/playlist_mig.go +++ b/pkg/services/sqlstore/migrations/playlist_mig.go @@ -32,4 +32,15 @@ func addPlaylistMigrations(mg *Migrator) { } mg.AddMigration("create playlist item table v2", NewAddTableMigration(playlistItemV2)) + + mg.AddMigration("Update playlist table charset", NewTableCharsetMigration("playlist", []*Column{ + {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "interval", Type: DB_NVarchar, Length: 255, Nullable: false}, + })) + + mg.AddMigration("Update playlist_item table charset", NewTableCharsetMigration("playlist_item", []*Column{ + {Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "value", Type: DB_Text, Nullable: false}, + {Name: "title", Type: DB_Text, Nullable: false}, + })) } diff --git a/pkg/services/sqlstore/migrations/plugin_setting.go b/pkg/services/sqlstore/migrations/plugin_setting.go index 0700ab67d2f..66a984b56d4 100644 --- a/pkg/services/sqlstore/migrations/plugin_setting.go +++ b/pkg/services/sqlstore/migrations/plugin_setting.go @@ -9,7 +9,7 @@ func addAppSettingsMigration(mg *Migrator) { Columns: []*Column{ {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, {Name: "org_id", Type: DB_BigInt, Nullable: true}, - {Name: "plugin_id", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "plugin_id", Type: DB_NVarchar, Length: 190, Nullable: false}, {Name: "enabled", Type: DB_Bool, Nullable: false}, {Name: "pinned", Type: DB_Bool, Nullable: false}, {Name: "json_data", Type: DB_Text, Nullable: true}, @@ -32,4 +32,10 @@ func addAppSettingsMigration(mg *Migrator) { Name: "plugin_version", Type: DB_NVarchar, Nullable: true, Length: 50, })) + mg.AddMigration("Update plugin_setting table charset", NewTableCharsetMigration("plugin_setting", []*Column{ + {Name: "plugin_id", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "json_data", Type: DB_Text, Nullable: true}, + {Name: "secure_json_data", Type: DB_Text, Nullable: true}, + {Name: "plugin_version", Type: DB_NVarchar, Nullable: true, Length: 50}, + })) } diff --git a/pkg/services/sqlstore/migrations/preferences_mig.go b/pkg/services/sqlstore/migrations/preferences_mig.go index 67a8169a7a8..b3822fe5239 100644 --- a/pkg/services/sqlstore/migrations/preferences_mig.go +++ b/pkg/services/sqlstore/migrations/preferences_mig.go @@ -29,4 +29,9 @@ func addPreferencesMigrations(mg *Migrator) { // create table mg.AddMigration("create preferences table v3", NewAddTableMigration(preferencesV2)) + + mg.AddMigration("Update preferences table charset", NewTableCharsetMigration("preferences", []*Column{ + {Name: "timezone", Type: DB_NVarchar, Length: 50, Nullable: false}, + {Name: "theme", Type: DB_NVarchar, Length: 20, Nullable: false}, + })) } diff --git a/pkg/services/sqlstore/migrations/quota_mig.go b/pkg/services/sqlstore/migrations/quota_mig.go index d877bb4c3c1..34f0da662a9 100644 --- a/pkg/services/sqlstore/migrations/quota_mig.go +++ b/pkg/services/sqlstore/migrations/quota_mig.go @@ -12,7 +12,7 @@ func addQuotaMigration(mg *Migrator) { {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, {Name: "org_id", Type: DB_BigInt, Nullable: true}, {Name: "user_id", Type: DB_BigInt, Nullable: true}, - {Name: "target", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "target", Type: DB_NVarchar, Length: 190, Nullable: false}, {Name: "limit", Type: DB_BigInt, Nullable: false}, {Name: "created", Type: DB_DateTime, Nullable: false}, {Name: "updated", Type: DB_DateTime, Nullable: false}, @@ -25,4 +25,8 @@ func addQuotaMigration(mg *Migrator) { //------- indexes ------------------ addTableIndicesMigrations(mg, "v1", quotaV1) + + mg.AddMigration("Update quota table charset", NewTableCharsetMigration("quota", []*Column{ + {Name: "target", Type: DB_NVarchar, Length: 190, Nullable: false}, + })) } diff --git a/pkg/services/sqlstore/migrations/temp_user.go b/pkg/services/sqlstore/migrations/temp_user.go index 3cae5d82472..5592ab7e4ad 100644 --- a/pkg/services/sqlstore/migrations/temp_user.go +++ b/pkg/services/sqlstore/migrations/temp_user.go @@ -35,4 +35,13 @@ func addTempUserMigrations(mg *Migrator) { // create table mg.AddMigration("create temp user table v1-7", NewAddTableMigration(tempUserV1)) addTableIndicesMigrations(mg, "v1-7", tempUserV1) + + mg.AddMigration("Update temp_user table charset", NewTableCharsetMigration("temp_user", []*Column{ + {Name: "email", Type: DB_NVarchar, Length: 255}, + {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "role", Type: DB_NVarchar, Length: 20, Nullable: true}, + {Name: "code", Type: DB_NVarchar, Length: 255}, + {Name: "status", Type: DB_Varchar, Length: 20}, + {Name: "remote_addr", Type: DB_Varchar, Length: 255, Nullable: true}, + })) } diff --git a/pkg/services/sqlstore/migrations/user_mig.go b/pkg/services/sqlstore/migrations/user_mig.go index 67446f64d1a..a12c4987792 100644 --- a/pkg/services/sqlstore/migrations/user_mig.go +++ b/pkg/services/sqlstore/migrations/user_mig.go @@ -8,8 +8,8 @@ func addUserMigrations(mg *Migrator) { Columns: []*Column{ {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, {Name: "version", Type: DB_Int, Nullable: false}, - {Name: "login", Type: DB_NVarchar, Length: 255, Nullable: false}, - {Name: "email", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "login", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "email", Type: DB_NVarchar, Length: 190, Nullable: false}, {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: true}, {Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true}, {Name: "salt", Type: DB_NVarchar, Length: 50, Nullable: true}, @@ -47,8 +47,8 @@ func addUserMigrations(mg *Migrator) { Columns: []*Column{ {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, {Name: "version", Type: DB_Int, Nullable: false}, - {Name: "login", Type: DB_NVarchar, Length: 255, Nullable: false}, - {Name: "email", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "login", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "email", Type: DB_NVarchar, Length: 190, Nullable: false}, {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: true}, {Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true}, {Name: "salt", Type: DB_NVarchar, Length: 50, Nullable: true}, @@ -92,4 +92,15 @@ func addUserMigrations(mg *Migrator) { mg.AddMigration("Add column help_flags1 to user table", NewAddColumnMigration(userV2, &Column{ Name: "help_flags1", Type: DB_BigInt, Nullable: false, Default: "0", })) + + mg.AddMigration("Update user table charset", NewTableCharsetMigration("user", []*Column{ + {Name: "login", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "email", Type: DB_NVarchar, Length: 190, Nullable: false}, + {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "salt", Type: DB_NVarchar, Length: 50, Nullable: true}, + {Name: "rands", Type: DB_NVarchar, Length: 50, Nullable: true}, + {Name: "company", Type: DB_NVarchar, Length: 255, Nullable: true}, + {Name: "theme", Type: DB_NVarchar, Length: 255, Nullable: true}, + })) } diff --git a/pkg/services/sqlstore/migrator/dialect.go b/pkg/services/sqlstore/migrator/dialect.go index 4473b560428..651405921d9 100644 --- a/pkg/services/sqlstore/migrator/dialect.go +++ b/pkg/services/sqlstore/migrator/dialect.go @@ -29,6 +29,7 @@ type Dialect interface { TableCheckSql(tableName string) (string, []interface{}) RenameTable(oldName string, newName string) string + UpdateTableSql(tableName string, columns []*Column) string } func NewDialect(name string) Dialect { @@ -102,7 +103,7 @@ func (b *BaseDialect) CreateTableSql(table *Table) string { sql = sql[:len(sql)-2] + ")" if b.dialect.SupportEngine() { - sql += " ENGINE=InnoDB DEFAULT CHARSET UTF8 " + sql += " ENGINE=InnoDB DEFAULT CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci" } sql += ";" @@ -160,3 +161,7 @@ func (db *BaseDialect) DropIndexSql(tableName string, index *Index) string { name = index.XName(tableName) return fmt.Sprintf("DROP INDEX %v ON %s", quote(name), quote(tableName)) } + +func (db *BaseDialect) UpdateTableSql(tableName string, columns []*Column) string { + return "-- NOT REQUIRED" +} diff --git a/pkg/services/sqlstore/migrator/migrations.go b/pkg/services/sqlstore/migrator/migrations.go index 7d97abb6bb9..2fec8825fa4 100644 --- a/pkg/services/sqlstore/migrator/migrations.go +++ b/pkg/services/sqlstore/migrator/migrations.go @@ -200,3 +200,17 @@ func (m *CopyTableDataMigration) IfTableExists(tableName string) *CopyTableDataM func (m *CopyTableDataMigration) Sql(d Dialect) string { return d.CopyTableData(m.sourceTable, m.targetTable, m.sourceCols, m.targetCols) } + +type TableCharsetMigration struct { + MigrationBase + tableName string + columns []*Column +} + +func NewTableCharsetMigration(tableName string, columns []*Column) *TableCharsetMigration { + return &TableCharsetMigration{tableName: tableName, columns: columns} +} + +func (m *TableCharsetMigration) Sql(d Dialect) string { + return d.UpdateTableSql(m.tableName, m.columns) +} diff --git a/pkg/services/sqlstore/migrator/mysql_dialect.go b/pkg/services/sqlstore/migrator/mysql_dialect.go index 48c2cd3ea55..1968558dbb8 100644 --- a/pkg/services/sqlstore/migrator/mysql_dialect.go +++ b/pkg/services/sqlstore/migrator/mysql_dialect.go @@ -1,6 +1,9 @@ package migrator -import "strconv" +import ( + "strconv" + "strings" +) type Mysql struct { BaseDialect @@ -76,6 +79,12 @@ func (db *Mysql) SqlType(c *Column) string { } else if hasLen1 { res += "(" + strconv.Itoa(c.Length) + ")" } + + switch c.Type { + case DB_Char, DB_Varchar, DB_NVarchar, DB_TinyText, DB_Text, DB_MediumText, DB_LongText: + res += " CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" + } + return res } @@ -84,3 +93,15 @@ func (db *Mysql) TableCheckSql(tableName string) (string, []interface{}) { sql := "SELECT `TABLE_NAME` from `INFORMATION_SCHEMA`.`TABLES` WHERE `TABLE_SCHEMA`=? and `TABLE_NAME`=?" return sql, args } + +func (db *Mysql) UpdateTableSql(tableName string, columns []*Column) string { + var statements = []string{} + + statements = append(statements, "DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci") + + for _, col := range columns { + statements = append(statements, "MODIFY "+col.StringNoPk(db)) + } + + return "ALTER TABLE " + db.Quote(tableName) + " " + strings.Join(statements, ", ") + ";" +} diff --git a/pkg/services/sqlstore/migrator/postgres_dialect.go b/pkg/services/sqlstore/migrator/postgres_dialect.go index 5500b9f1684..92109efdfab 100644 --- a/pkg/services/sqlstore/migrator/postgres_dialect.go +++ b/pkg/services/sqlstore/migrator/postgres_dialect.go @@ -3,6 +3,7 @@ package migrator import ( "fmt" "strconv" + "strings" ) type Postgres struct { @@ -112,3 +113,13 @@ func (db *Postgres) DropIndexSql(tableName string, index *Index) string { idxName := index.XName(tableName) return fmt.Sprintf("DROP INDEX %v", quote(idxName)) } + +func (db *Postgres) UpdateTableSql(tableName string, columns []*Column) string { + var statements = []string{} + + for _, col := range columns { + statements = append(statements, "ALTER "+db.QuoteStr()+col.Name+db.QuoteStr()+" TYPE "+db.SqlType(col)) + } + + return "ALTER TABLE " + db.Quote(tableName) + " " + strings.Join(statements, ", ") + ";" +} diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go index d1f555d2d7f..a2106ddbffd 100644 --- a/pkg/services/sqlstore/sqlstore.go +++ b/pkg/services/sqlstore/sqlstore.go @@ -114,7 +114,7 @@ func getEngine() (*xorm.Engine, error) { protocol = "unix" } - cnnstr = fmt.Sprintf("%s:%s@%s(%s)/%s?charset=utf8", + cnnstr = fmt.Sprintf("%s:%s@%s(%s)/%s?charset=utf8mb4", DbCfg.User, DbCfg.Pwd, protocol, DbCfg.Host, DbCfg.Name) if DbCfg.SslMode == "true" || DbCfg.SslMode == "skip-verify" { diff --git a/pkg/services/sqlstore/sqlutil/sqlutil.go b/pkg/services/sqlstore/sqlutil/sqlutil.go index 075b56c1c18..68ced55722c 100644 --- a/pkg/services/sqlstore/sqlutil/sqlutil.go +++ b/pkg/services/sqlstore/sqlutil/sqlutil.go @@ -12,7 +12,7 @@ type TestDB struct { } var TestDB_Sqlite3 = TestDB{DriverName: "sqlite3", ConnStr: ":memory:?_loc=Local"} -var TestDB_Mysql = TestDB{DriverName: "mysql", ConnStr: "grafana:password@tcp(localhost:3306)/grafana_tests?charset=utf8"} +var TestDB_Mysql = TestDB{DriverName: "mysql", ConnStr: "grafana:password@tcp(localhost:3306)/grafana_tests?charset=utf8mb4"} var TestDB_Postgres = TestDB{DriverName: "postgres", ConnStr: "user=grafanatest password=grafanatest host=localhost port=5432 dbname=grafanatest sslmode=disable"} func CleanDB(x *xorm.Engine) {