More work on sql schema and migrations, starting to get somewhere

pkg/models/account.go

@@ -10,6 +10,7 @@ var (
 	ErrAccountNotFound = errors.New("Account not found")
 )
 
+// Directly mapped to db schema, Do not change field names lighly
 type Account struct {
 	Id              int64
 	Login           string `xorm:"UNIQUE NOT NULL"`

pkg/services/sqlstore/migrations/builder.go

@@ -1,54 +1,122 @@
 package migrations
 
-type migration struct {
-	desc        string
-	sqlite      string
-	mysql       string
-	verifyTable string
-}
-
-type columnType string
-
-const (
-	DB_TYPE_STRING columnType = "String"
+import (
+	"fmt"
+	"strings"
 )
 
-func (m *migration) getSql(dbType string) string {
-	switch dbType {
-	case "mysql":
+const (
+	POSTGRES = "postgres"
+	SQLITE   = "sqlite3"
+	MYSQL    = "mysql"
+)
+
+type Migration interface {
+	Sql(dialect Dialect) string
+}
+
+type ColumnType string
+
+const (
+	DB_TYPE_STRING ColumnType = "String"
+)
+
+type MigrationBase struct {
+	desc string
+}
+
+type RawSqlMigration struct {
+	MigrationBase
+
+	sqlite string
+	mysql  string
+}
+
+func (m *RawSqlMigration) Sql(dialect Dialect) string {
+	switch dialect.DriverName() {
+	case MYSQL:
 		return m.mysql
-	case "sqlite3":
+	case SQLITE:
 		return m.sqlite
 	}
 
 	panic("db type not supported")
 }
 
-type migrationBuilder struct {
-	migration *migration
+func (m *RawSqlMigration) Sqlite(sql string) *RawSqlMigration {
+	m.sqlite = sql
+	return m
 }
 
-func (b *migrationBuilder) sqlite(sql string) *migrationBuilder {
-	b.migration.sqlite = sql
-	return b
+func (m *RawSqlMigration) Mysql(sql string) *RawSqlMigration {
+	m.mysql = sql
+	return m
 }
 
-func (b *migrationBuilder) mysql(sql string) *migrationBuilder {
-	b.migration.mysql = sql
-	return b
+func (m *RawSqlMigration) Desc(desc string) *RawSqlMigration {
+	m.desc = desc
+	return m
 }
 
-func (b *migrationBuilder) verifyTable(name string) *migrationBuilder {
-	b.migration.verifyTable = name
-	return b
+type AddColumnMigration struct {
+	MigrationBase
+	tableName  string
+	columnName string
+	columnType ColumnType
+	length     int
 }
 
-func (b *migrationBuilder) add() *migrationBuilder {
-	migrationList = append(migrationList, b.migration)
-	return b
+func (m *AddColumnMigration) Table(tableName string) *AddColumnMigration {
+	m.tableName = tableName
+	return m
 }
 
-func (b *migrationBuilder) desc(desc string) *migrationBuilder {
-	b.migration = &migration{desc: desc}
-	return b
+func (m *AddColumnMigration) Length(length int) *AddColumnMigration {
+	m.length = length
+	return m
+}
+
+func (m *AddColumnMigration) Column(columnName string) *AddColumnMigration {
+	m.columnName = columnName
+	return m
+}
+
+func (m *AddColumnMigration) Type(columnType ColumnType) *AddColumnMigration {
+	m.columnType = columnType
+	return m
+}
+
+func (m *AddColumnMigration) Sql(dialect Dialect) string {
+	return fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", m.tableName, m.columnName, dialect.ToDBTypeSql(m.columnType, m.length))
+}
+
+func (m *AddColumnMigration) Desc(desc string) *AddColumnMigration {
+	m.desc = desc
+	return m
+}
+
+type AddIndexMigration struct {
+	MigrationBase
+	tableName string
+	columns   string
+	indexName string
+}
+
+func (m *AddIndexMigration) Name(name string) *AddIndexMigration {
+	m.indexName = name
+	return m
+}
+
+func (m *AddIndexMigration) Table(tableName string) *AddIndexMigration {
+	m.tableName = tableName
+	return m
+}
+
+func (m *AddIndexMigration) Columns(columns ...string) *AddIndexMigration {
+	m.columns = strings.Join(columns, ",")
+	return m
+}
+
+func (m *AddIndexMigration) Sql(dialect Dialect) string {
+	return fmt.Sprintf("CREATE UNIQUE INDEX %s ON %s(%s)", m.indexName, m.tableName, m.columns)
 }

pkg/services/sqlstore/migrations/dialect.go

@@ -0,0 +1,52 @@
+package migrations
+
+import "fmt"
+
+type Dialect interface {
+	DriverName() string
+	ToDBTypeSql(columnType ColumnType, length int) string
+	TableCheckSql(tableName string) (string, []interface{})
+}
+
+type Sqlite3 struct {
+}
+
+type Mysql struct {
+}
+
+func (db *Sqlite3) DriverName() string {
+	return SQLITE
+}
+
+func (db *Mysql) DriverName() string {
+	return MYSQL
+}
+
+func (db *Sqlite3) ToDBTypeSql(columnType ColumnType, length int) string {
+	switch columnType {
+	case DB_TYPE_STRING:
+		return "TEXT"
+	}
+
+	panic("Unsupported db type")
+}
+
+func (db *Mysql) ToDBTypeSql(columnType ColumnType, length int) string {
+	switch columnType {
+	case DB_TYPE_STRING:
+		return fmt.Sprintf("NVARCHAR(%d)", length)
+	}
+
+	panic("Unsupported db type")
+}
+
+func (db *Sqlite3) TableCheckSql(tableName string) (string, []interface{}) {
+	args := []interface{}{tableName}
+	return "SELECT name FROM sqlite_master WHERE type='table' and name = ?", args
+}
+
+func (db *Mysql) TableCheckSql(tableName string) (string, []interface{}) {
+	args := []interface{}{"grafana", tableName}
+	sql := "SELECT `TABLE_NAME` from `INFORMATION_SCHEMA`.`TABLES` WHERE `TABLE_SCHEMA`=? and `TABLE_NAME`=?"
+	return sql, args
+}

pkg/services/sqlstore/migrations/engine.go

@@ -1,19 +1,15 @@
 package migrations
 
 import (
-	"errors"
-	"fmt"
-
-	"github.com/torkelo/grafana-pro/pkg/services/sqlstore/sqlsyntax"
-
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/go-xorm/xorm"
 	_ "github.com/lib/pq"
 	_ "github.com/mattn/go-sqlite3"
+	"github.com/torkelo/grafana-pro/pkg/log"
 )
 
 var x *xorm.Engine
-var dialect sqlsyntax.Dialect
+var dialect Dialect
 
 func getSchemaVersion() (int, error) {
 	exists, err := x.IsTableExist(new(SchemaVersion))
@@ -36,14 +32,16 @@ func getSchemaVersion() (int, error) {
 func setEngineAndDialect(engine *xorm.Engine) {
 	x = engine
 	switch x.DriverName() {
-	case "mysql":
-		dialect = new(sqlsyntax.Mysql)
-	case "sqlite3":
-		dialect = new(sqlsyntax.Sqlite3)
+	case MYSQL:
+		dialect = new(Mysql)
+	case SQLITE:
+		dialect = new(Sqlite3)
 	}
 }
 
 func StartMigration(engine *xorm.Engine) error {
+	log.Info("Starting database schema migration: DB: %v", engine.DriverName())
+
 	setEngineAndDialect(engine)
 
 	_, err := getSchemaVersion()
@@ -60,9 +58,9 @@ func StartMigration(engine *xorm.Engine) error {
 	return nil
 }
 
-func execMigration(m *migration) error {
+func execMigration(m Migration) error {
 	err := inTransaction(func(sess *xorm.Session) error {
-		_, err := sess.Exec(m.getSql(x.DriverName()))
+		_, err := sess.Exec(m.Sql(dialect))
 		if err != nil {
 			return err
 		}
@@ -73,17 +71,6 @@ func execMigration(m *migration) error {
 		return err
 	}
 
-	return verifyMigration(m)
-}
-
-func verifyMigration(m *migration) error {
-	if m.verifyTable != "" {
-		sqlStr, args := dialect.TableCheckSql(m.verifyTable)
-		results, err := x.Query(sqlStr, args...)
-		if err != nil || len(results) == 0 {
-			return errors.New(fmt.Sprintf("Verify failed: table %v does not exist", m.verifyTable))
-		}
-	}
 	return nil
 }
 

pkg/services/sqlstore/migrations/migrations.go

@@ -1,27 +1,48 @@
 package migrations
 
-var migrationList []*migration
+var migrationList []Migration
+
+// Id              int64
+// Login           string `xorm:"UNIQUE NOT NULL"`
+// Email           string `xorm:"UNIQUE NOT NULL"`
+// Name            string
+// FullName        string
+// Password        string
+// IsAdmin         bool
+// Salt            string `xorm:"VARCHAR(10)"`
+// Company         string
+// NextDashboardId int
+// UsingAccountId  int64
+// Created         time.Time
+// Updated         time.Time
 
 func init() {
-	new(migrationBuilder).
-		// ------------------------------
-		desc("Create account table").
-		sqlite(`
+	// ------------------------------
+	addMigration(new(RawSqlMigration).Desc("Create account table").
+		Sqlite(`
 		  CREATE TABLE account (
-		  	id INTEGER PRIMARY KEY AUTOINCREMENT
+		  	id INTEGER PRIMARY KEY AUTOINCREMENT,
+		  	login TEXT NOT NULL,
+		  	email TEXT NOT NULL
 			)
 		`).
-		mysql(`
+		Mysql(`
 		  CREATE TABLE account (
-		  	id BIGINT NOT NULL AUTO_INCREMENT, PRIMARY KEY (id)
+		  	id BIGINT NOT NULL AUTO_INCREMENT, PRIMARY KEY (id),
+		  	login  VARCHAR(255) NOT NULL,
+		  	email  VARCHAR(255) NOT NULL
 		  )
-		`).
-		verifyTable("account").add()
+		`))
 	// ------------------------------
-	//		desc("Add name column to account table").
-	// table("account").addColumn("name").colType(DB_TYPE_STRING)
-	// sqlite("ALTER TABLE account ADD COLUMN name TEXT").
-	// mysql("ALTER TABLE account ADD COLUMN name NVARCHAR(255)").
+	addMigration(new(AddIndexMigration).
+		Name("UIX_account_login").Table("account").Columns("login"))
+	// ------------------------------
+	addMigration(new(AddColumnMigration).Desc("Add name column").
+		Table("account").Column("name").Type(DB_TYPE_STRING).Length(255))
+}
+
+func addMigration(m Migration) {
+	migrationList = append(migrationList, m)
 }
 
 type SchemaVersion struct {

pkg/services/sqlstore/migrations/migrations_test.go

@@ -2,6 +2,7 @@ package migrations
 
 import (
 	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/go-xorm/xorm"
@@ -27,10 +28,12 @@ func cleanDB(x *xorm.Engine) {
 	}
 }
 
-func TestMigrationsSqlite(t *testing.T) {
+var indexTypes = []string{"Unknown", "", "UNIQUE"}
+
+func TestMigrations(t *testing.T) {
 	testDBs := [][]string{
+		//[]string{"mysql", "grafana:password@tcp(localhost:3306)/grafana_tests?charset=utf8"},
 		[]string{"sqlite3", ":memory:"},
-		[]string{"mysql", "grafana:password@tcp(localhost:3306)/grafana_tests?charset=utf8"},
 	}
 
 	for _, testDB := range testDBs {
@@ -43,13 +46,28 @@ func TestMigrationsSqlite(t *testing.T) {
 				cleanDB(x)
 			}
 
-			StartMigration(x)
+			err = StartMigration(x)
+			So(err, ShouldBeNil)
 
 			tables, err := x.DBMetas()
 			So(err, ShouldBeNil)
 
 			So(len(tables), ShouldEqual, 2)
-		})
+			fmt.Printf("\nDB Schema after migration: table count: %v\n", len(tables))
 
+			for _, table := range tables {
+				fmt.Printf("\nTable: %v \n", table.Name)
+				for _, column := range table.Columns() {
+					fmt.Printf("\t %v \n", column.String(x.Dialect()))
+				}
+
+				if len(table.Indexes) > 0 {
+					fmt.Printf("\n\tIndexes:\n")
+					for _, index := range table.Indexes {
+						fmt.Printf("\t %v (%v) %v \n", index.Name, strings.Join(index.Cols, ","), indexTypes[index.Type])
+					}
+				}
+			}
+		})
 	}
 }

pkg/services/sqlstore/sqlsyntax/dialect.go

@@ -1,31 +0,0 @@
-package sqlsyntax
-
-type Dialect interface {
-	DBType() string
-	TableCheckSql(tableName string) (string, []interface{})
-}
-
-type Sqlite3 struct {
-}
-
-type Mysql struct {
-}
-
-func (db *Sqlite3) DBType() string {
-	return "sqlite3"
-}
-
-func (db *Mysql) DBType() string {
-	return "mysql"
-}
-
-func (db *Sqlite3) TableCheckSql(tableName string) (string, []interface{}) {
-	args := []interface{}{tableName}
-	return "SELECT name FROM sqlite_master WHERE type='table' and name = ?", args
-}
-
-func (db *Mysql) TableCheckSql(tableName string) (string, []interface{}) {
-	args := []interface{}{"grafana", tableName}
-	sql := "SELECT `TABLE_NAME` from `INFORMATION_SCHEMA`.`TABLES` WHERE `TABLE_SCHEMA`=? and `TABLE_NAME`=?"
-	return sql, args
-}