From 8bfed7508cad2811504603c4a438335d45dfd048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Sun, 18 Jan 2015 14:51:51 +0100 Subject: [PATCH] More work on sql schema and migrations, starting to get somewhere --- pkg/models/account.go | 1 + pkg/services/sqlstore/migrations/builder.go | 132 +++++++++++++----- pkg/services/sqlstore/migrations/dialect.go | 52 +++++++ pkg/services/sqlstore/migrations/engine.go | 33 ++--- .../sqlstore/migrations/migrations.go | 49 +++++-- .../sqlstore/migrations/migrations_test.go | 26 +++- pkg/services/sqlstore/sqlsyntax/dialect.go | 31 ---- 7 files changed, 220 insertions(+), 104 deletions(-) create mode 100644 pkg/services/sqlstore/migrations/dialect.go delete mode 100644 pkg/services/sqlstore/sqlsyntax/dialect.go diff --git a/pkg/models/account.go b/pkg/models/account.go index 36277a4b9c9..0f7dd876949 100644 --- a/pkg/models/account.go +++ b/pkg/models/account.go @@ -10,6 +10,7 @@ var ( ErrAccountNotFound = errors.New("Account not found") ) +// Directly mapped to db schema, Do not change field names lighly type Account struct { Id int64 Login string `xorm:"UNIQUE NOT NULL"` diff --git a/pkg/services/sqlstore/migrations/builder.go b/pkg/services/sqlstore/migrations/builder.go index 0d5ba6856dd..0b1accae8dd 100644 --- a/pkg/services/sqlstore/migrations/builder.go +++ b/pkg/services/sqlstore/migrations/builder.go @@ -1,54 +1,122 @@ package migrations -type migration struct { - desc string - sqlite string - mysql string - verifyTable string -} - -type columnType string - -const ( - DB_TYPE_STRING columnType = "String" +import ( + "fmt" + "strings" ) -func (m *migration) getSql(dbType string) string { - switch dbType { - case "mysql": +const ( + POSTGRES = "postgres" + SQLITE = "sqlite3" + MYSQL = "mysql" +) + +type Migration interface { + Sql(dialect Dialect) string +} + +type ColumnType string + +const ( + DB_TYPE_STRING ColumnType = "String" +) + +type MigrationBase struct { + desc string +} + +type RawSqlMigration struct { + MigrationBase + + sqlite string + mysql string +} + +func (m *RawSqlMigration) Sql(dialect Dialect) string { + switch dialect.DriverName() { + case MYSQL: return m.mysql - case "sqlite3": + case SQLITE: return m.sqlite } panic("db type not supported") } -type migrationBuilder struct { - migration *migration +func (m *RawSqlMigration) Sqlite(sql string) *RawSqlMigration { + m.sqlite = sql + return m } -func (b *migrationBuilder) sqlite(sql string) *migrationBuilder { - b.migration.sqlite = sql - return b +func (m *RawSqlMigration) Mysql(sql string) *RawSqlMigration { + m.mysql = sql + return m } -func (b *migrationBuilder) mysql(sql string) *migrationBuilder { - b.migration.mysql = sql - return b +func (m *RawSqlMigration) Desc(desc string) *RawSqlMigration { + m.desc = desc + return m } -func (b *migrationBuilder) verifyTable(name string) *migrationBuilder { - b.migration.verifyTable = name - return b +type AddColumnMigration struct { + MigrationBase + tableName string + columnName string + columnType ColumnType + length int } -func (b *migrationBuilder) add() *migrationBuilder { - migrationList = append(migrationList, b.migration) - return b +func (m *AddColumnMigration) Table(tableName string) *AddColumnMigration { + m.tableName = tableName + return m } -func (b *migrationBuilder) desc(desc string) *migrationBuilder { - b.migration = &migration{desc: desc} - return b +func (m *AddColumnMigration) Length(length int) *AddColumnMigration { + m.length = length + return m +} + +func (m *AddColumnMigration) Column(columnName string) *AddColumnMigration { + m.columnName = columnName + return m +} + +func (m *AddColumnMigration) Type(columnType ColumnType) *AddColumnMigration { + m.columnType = columnType + return m +} + +func (m *AddColumnMigration) Sql(dialect Dialect) string { + return fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", m.tableName, m.columnName, dialect.ToDBTypeSql(m.columnType, m.length)) +} + +func (m *AddColumnMigration) Desc(desc string) *AddColumnMigration { + m.desc = desc + return m +} + +type AddIndexMigration struct { + MigrationBase + tableName string + columns string + indexName string +} + +func (m *AddIndexMigration) Name(name string) *AddIndexMigration { + m.indexName = name + return m +} + +func (m *AddIndexMigration) Table(tableName string) *AddIndexMigration { + m.tableName = tableName + return m +} + +func (m *AddIndexMigration) Columns(columns ...string) *AddIndexMigration { + m.columns = strings.Join(columns, ",") + return m +} + +func (m *AddIndexMigration) Sql(dialect Dialect) string { + return fmt.Sprintf("CREATE UNIQUE INDEX %s ON %s(%s)", m.indexName, m.tableName, m.columns) } diff --git a/pkg/services/sqlstore/migrations/dialect.go b/pkg/services/sqlstore/migrations/dialect.go new file mode 100644 index 00000000000..2a7c7c569e0 --- /dev/null +++ b/pkg/services/sqlstore/migrations/dialect.go @@ -0,0 +1,52 @@ +package migrations + +import "fmt" + +type Dialect interface { + DriverName() string + ToDBTypeSql(columnType ColumnType, length int) string + TableCheckSql(tableName string) (string, []interface{}) +} + +type Sqlite3 struct { +} + +type Mysql struct { +} + +func (db *Sqlite3) DriverName() string { + return SQLITE +} + +func (db *Mysql) DriverName() string { + return MYSQL +} + +func (db *Sqlite3) ToDBTypeSql(columnType ColumnType, length int) string { + switch columnType { + case DB_TYPE_STRING: + return "TEXT" + } + + panic("Unsupported db type") +} + +func (db *Mysql) ToDBTypeSql(columnType ColumnType, length int) string { + switch columnType { + case DB_TYPE_STRING: + return fmt.Sprintf("NVARCHAR(%d)", length) + } + + panic("Unsupported db type") +} + +func (db *Sqlite3) TableCheckSql(tableName string) (string, []interface{}) { + args := []interface{}{tableName} + return "SELECT name FROM sqlite_master WHERE type='table' and name = ?", args +} + +func (db *Mysql) TableCheckSql(tableName string) (string, []interface{}) { + args := []interface{}{"grafana", tableName} + sql := "SELECT `TABLE_NAME` from `INFORMATION_SCHEMA`.`TABLES` WHERE `TABLE_SCHEMA`=? and `TABLE_NAME`=?" + return sql, args +} diff --git a/pkg/services/sqlstore/migrations/engine.go b/pkg/services/sqlstore/migrations/engine.go index 9260bab8259..b6362ff2abf 100644 --- a/pkg/services/sqlstore/migrations/engine.go +++ b/pkg/services/sqlstore/migrations/engine.go @@ -1,19 +1,15 @@ package migrations import ( - "errors" - "fmt" - - "github.com/torkelo/grafana-pro/pkg/services/sqlstore/sqlsyntax" - _ "github.com/go-sql-driver/mysql" "github.com/go-xorm/xorm" _ "github.com/lib/pq" _ "github.com/mattn/go-sqlite3" + "github.com/torkelo/grafana-pro/pkg/log" ) var x *xorm.Engine -var dialect sqlsyntax.Dialect +var dialect Dialect func getSchemaVersion() (int, error) { exists, err := x.IsTableExist(new(SchemaVersion)) @@ -36,14 +32,16 @@ func getSchemaVersion() (int, error) { func setEngineAndDialect(engine *xorm.Engine) { x = engine switch x.DriverName() { - case "mysql": - dialect = new(sqlsyntax.Mysql) - case "sqlite3": - dialect = new(sqlsyntax.Sqlite3) + case MYSQL: + dialect = new(Mysql) + case SQLITE: + dialect = new(Sqlite3) } } func StartMigration(engine *xorm.Engine) error { + log.Info("Starting database schema migration: DB: %v", engine.DriverName()) + setEngineAndDialect(engine) _, err := getSchemaVersion() @@ -60,9 +58,9 @@ func StartMigration(engine *xorm.Engine) error { return nil } -func execMigration(m *migration) error { +func execMigration(m Migration) error { err := inTransaction(func(sess *xorm.Session) error { - _, err := sess.Exec(m.getSql(x.DriverName())) + _, err := sess.Exec(m.Sql(dialect)) if err != nil { return err } @@ -73,17 +71,6 @@ func execMigration(m *migration) error { return err } - return verifyMigration(m) -} - -func verifyMigration(m *migration) error { - if m.verifyTable != "" { - sqlStr, args := dialect.TableCheckSql(m.verifyTable) - results, err := x.Query(sqlStr, args...) - if err != nil || len(results) == 0 { - return errors.New(fmt.Sprintf("Verify failed: table %v does not exist", m.verifyTable)) - } - } return nil } diff --git a/pkg/services/sqlstore/migrations/migrations.go b/pkg/services/sqlstore/migrations/migrations.go index 94c42ad6bc4..0e261ecb01c 100644 --- a/pkg/services/sqlstore/migrations/migrations.go +++ b/pkg/services/sqlstore/migrations/migrations.go @@ -1,27 +1,48 @@ package migrations -var migrationList []*migration +var migrationList []Migration + +// Id int64 +// Login string `xorm:"UNIQUE NOT NULL"` +// Email string `xorm:"UNIQUE NOT NULL"` +// Name string +// FullName string +// Password string +// IsAdmin bool +// Salt string `xorm:"VARCHAR(10)"` +// Company string +// NextDashboardId int +// UsingAccountId int64 +// Created time.Time +// Updated time.Time func init() { - new(migrationBuilder). - // ------------------------------ - desc("Create account table"). - sqlite(` + // ------------------------------ + addMigration(new(RawSqlMigration).Desc("Create account table"). + Sqlite(` CREATE TABLE account ( - id INTEGER PRIMARY KEY AUTOINCREMENT + id INTEGER PRIMARY KEY AUTOINCREMENT, + login TEXT NOT NULL, + email TEXT NOT NULL ) `). - mysql(` + Mysql(` CREATE TABLE account ( - id BIGINT NOT NULL AUTO_INCREMENT, PRIMARY KEY (id) + id BIGINT NOT NULL AUTO_INCREMENT, PRIMARY KEY (id), + login VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL ) - `). - verifyTable("account").add() + `)) // ------------------------------ - // desc("Add name column to account table"). - // table("account").addColumn("name").colType(DB_TYPE_STRING) - // sqlite("ALTER TABLE account ADD COLUMN name TEXT"). - // mysql("ALTER TABLE account ADD COLUMN name NVARCHAR(255)"). + addMigration(new(AddIndexMigration). + Name("UIX_account_login").Table("account").Columns("login")) + // ------------------------------ + addMigration(new(AddColumnMigration).Desc("Add name column"). + Table("account").Column("name").Type(DB_TYPE_STRING).Length(255)) +} + +func addMigration(m Migration) { + migrationList = append(migrationList, m) } type SchemaVersion struct { diff --git a/pkg/services/sqlstore/migrations/migrations_test.go b/pkg/services/sqlstore/migrations/migrations_test.go index d30d360bca9..247ebd6ca68 100644 --- a/pkg/services/sqlstore/migrations/migrations_test.go +++ b/pkg/services/sqlstore/migrations/migrations_test.go @@ -2,6 +2,7 @@ package migrations import ( "fmt" + "strings" "testing" "github.com/go-xorm/xorm" @@ -27,10 +28,12 @@ func cleanDB(x *xorm.Engine) { } } -func TestMigrationsSqlite(t *testing.T) { +var indexTypes = []string{"Unknown", "", "UNIQUE"} + +func TestMigrations(t *testing.T) { testDBs := [][]string{ + //[]string{"mysql", "grafana:password@tcp(localhost:3306)/grafana_tests?charset=utf8"}, []string{"sqlite3", ":memory:"}, - []string{"mysql", "grafana:password@tcp(localhost:3306)/grafana_tests?charset=utf8"}, } for _, testDB := range testDBs { @@ -43,13 +46,28 @@ func TestMigrationsSqlite(t *testing.T) { cleanDB(x) } - StartMigration(x) + err = StartMigration(x) + So(err, ShouldBeNil) tables, err := x.DBMetas() So(err, ShouldBeNil) So(len(tables), ShouldEqual, 2) - }) + fmt.Printf("\nDB Schema after migration: table count: %v\n", len(tables)) + for _, table := range tables { + fmt.Printf("\nTable: %v \n", table.Name) + for _, column := range table.Columns() { + fmt.Printf("\t %v \n", column.String(x.Dialect())) + } + + if len(table.Indexes) > 0 { + fmt.Printf("\n\tIndexes:\n") + for _, index := range table.Indexes { + fmt.Printf("\t %v (%v) %v \n", index.Name, strings.Join(index.Cols, ","), indexTypes[index.Type]) + } + } + } + }) } } diff --git a/pkg/services/sqlstore/sqlsyntax/dialect.go b/pkg/services/sqlstore/sqlsyntax/dialect.go deleted file mode 100644 index 6104f885c86..00000000000 --- a/pkg/services/sqlstore/sqlsyntax/dialect.go +++ /dev/null @@ -1,31 +0,0 @@ -package sqlsyntax - -type Dialect interface { - DBType() string - TableCheckSql(tableName string) (string, []interface{}) -} - -type Sqlite3 struct { -} - -type Mysql struct { -} - -func (db *Sqlite3) DBType() string { - return "sqlite3" -} - -func (db *Mysql) DBType() string { - return "mysql" -} - -func (db *Sqlite3) TableCheckSql(tableName string) (string, []interface{}) { - args := []interface{}{tableName} - return "SELECT name FROM sqlite_master WHERE type='table' and name = ?", args -} - -func (db *Mysql) TableCheckSql(tableName string) (string, []interface{}) { - args := []interface{}{"grafana", tableName} - sql := "SELECT `TABLE_NAME` from `INFORMATION_SCHEMA`.`TABLES` WHERE `TABLE_SCHEMA`=? and `TABLE_NAME`=?" - return sql, args -}