a lot of work on database schema and migration setup, postgres now works, every integration test passes for all database types, only token table left to do

pkg/services/sqlstore/migrator/builder.go

@@ -0,0 +1,131 @@
+package migrator
+
+import (
+	"fmt"
+	"strings"
+)
+
+type MigrationBase struct {
+	id string
+}
+
+func (m *MigrationBase) Id() string {
+	return m.id
+}
+
+func (m *MigrationBase) SetId(id string) {
+	m.id = id
+}
+
+type RawSqlMigration struct {
+	MigrationBase
+
+	sqlite string
+	mysql  string
+}
+
+func (m *RawSqlMigration) Sql(dialect Dialect) string {
+	switch dialect.DriverName() {
+	case MYSQL:
+		return m.mysql
+	case SQLITE:
+		return m.sqlite
+	}
+
+	panic("db type not supported")
+}
+
+func (m *RawSqlMigration) Sqlite(sql string) *RawSqlMigration {
+	m.sqlite = sql
+	return m
+}
+
+func (m *RawSqlMigration) Mysql(sql string) *RawSqlMigration {
+	m.mysql = sql
+	return m
+}
+
+type AddColumnMigration struct {
+	MigrationBase
+	tableName string
+	column    *Column
+}
+
+func (m *AddColumnMigration) Table(tableName string) *AddColumnMigration {
+	m.tableName = tableName
+	return m
+}
+
+func (m *AddColumnMigration) Column(col *Column) *AddColumnMigration {
+	m.column = col
+	return m
+}
+
+func (m *AddColumnMigration) Sql(dialect Dialect) string {
+	return dialect.AddColumnSql(m.tableName, m.column)
+}
+
+type AddIndexMigration struct {
+	MigrationBase
+	tableName string
+	index     Index
+}
+
+func (m *AddIndexMigration) Name(name string) *AddIndexMigration {
+	m.index.Name = name
+	return m
+}
+
+func (m *AddIndexMigration) Table(tableName string) *AddIndexMigration {
+	m.tableName = tableName
+	return m
+}
+
+func (m *AddIndexMigration) Unique() *AddIndexMigration {
+	m.index.Type = UniqueIndex
+	return m
+}
+
+func (m *AddIndexMigration) Columns(columns ...string) *AddIndexMigration {
+	m.index.Cols = columns
+	return m
+}
+
+func (m *AddIndexMigration) Sql(dialect Dialect) string {
+	if m.index.Name == "" {
+		m.index.Name = fmt.Sprintf("%s", strings.Join(m.index.Cols, "_"))
+	}
+	return dialect.CreateIndexSql(m.tableName, &m.index)
+}
+
+type AddTableMigration struct {
+	MigrationBase
+	table Table
+}
+
+func (m *AddTableMigration) Sql(d Dialect) string {
+	return d.CreateTableSql(&m.table)
+}
+
+func (m *AddTableMigration) Name(name string) *AddTableMigration {
+	m.table.Name = name
+	return m
+}
+
+func (m *AddTableMigration) WithColumns(columns ...*Column) *AddTableMigration {
+	for _, col := range columns {
+		m.table.Columns = append(m.table.Columns, col)
+		if col.IsPrimaryKey {
+			m.table.PrimaryKeys = append(m.table.PrimaryKeys, col.Name)
+		}
+	}
+	return m
+}
+
+func (m *AddTableMigration) WithColumn(col *Column) *AddTableMigration {
+	m.table.Columns = append(m.table.Columns, col)
+	if col.IsPrimaryKey {
+		m.table.PrimaryKeys = append(m.table.PrimaryKeys, col.Name)
+	}
+	return m
+}

pkg/services/sqlstore/migrator/column.go

@@ -0,0 +1,62 @@
+package migrator
+
+// Notice
+// code based on parts from from https://github.com/go-xorm/core/blob/3e0fa232ab5c90996406c0cd7ae86ad0e5ecf85f/column.go
+
+type Column struct {
+	Name            string
+	Type            string
+	Length          int
+	Length2         int
+	Nullable        bool
+	IsPrimaryKey    bool
+	IsAutoIncrement bool
+	Default         string
+}
+
+func (col *Column) String(d Dialect) string {
+	sql := d.QuoteStr() + col.Name + d.QuoteStr() + " "
+
+	sql += d.SqlType(col) + " "
+
+	if col.IsPrimaryKey {
+		sql += "PRIMARY KEY "
+		if col.IsAutoIncrement {
+			sql += d.AutoIncrStr() + " "
+		}
+	}
+
+	if d.ShowCreateNull() {
+		if col.Nullable {
+			sql += "NULL "
+		} else {
+			sql += "NOT NULL "
+		}
+	}
+
+	if col.Default != "" {
+		sql += "DEFAULT " + col.Default + " "
+	}
+
+	return sql
+}
+
+func (col *Column) StringNoPk(d Dialect) string {
+	sql := d.QuoteStr() + col.Name + d.QuoteStr() + " "
+
+	sql += d.SqlType(col) + " "
+
+	if d.ShowCreateNull() {
+		if col.Nullable {
+			sql += "NULL "
+		} else {
+			sql += "NOT NULL "
+		}
+	}
+
+	if col.Default != "" {
+		sql += "DEFAULT " + col.Default + " "
+	}
+
+	return sql
+}

pkg/services/sqlstore/migrator/dialect.go

@@ -0,0 +1,110 @@
+package migrator
+
+import (
+	"fmt"
+	"strings"
+)
+
+type Dialect interface {
+	DriverName() string
+	QuoteStr() string
+	Quote(string) string
+	AndStr() string
+	AutoIncrStr() string
+	OrStr() string
+	EqStr() string
+	ShowCreateNull() bool
+	SqlType(col *Column) string
+
+	CreateIndexSql(tableName string, index *Index) string
+	CreateTableSql(table *Table) string
+	AddColumnSql(tableName string, Col *Column) string
+
+	TableCheckSql(tableName string) (string, []interface{})
+}
+
+func NewDialect(name string) Dialect {
+	switch name {
+	case MYSQL:
+		return NewMysqlDialect()
+	case SQLITE:
+		return NewSqlite3Dialect()
+	case POSTGRES:
+		return NewPostgresDialect()
+	}
+
+	panic("Unsupported database type: " + name)
+}
+
+type BaseDialect struct {
+	dialect    Dialect
+	driverName string
+}
+
+func (d *BaseDialect) DriverName() string {
+	return d.driverName
+}
+
+func (b *BaseDialect) ShowCreateNull() bool {
+	return true
+}
+
+func (b *BaseDialect) AndStr() string {
+	return "AND"
+}
+
+func (b *BaseDialect) OrStr() string {
+	return "OR"
+}
+
+func (b *BaseDialect) EqStr() string {
+	return "="
+}
+
+func (b *BaseDialect) CreateTableSql(table *Table) string {
+	var sql string
+	sql = "CREATE TABLE IF NOT EXISTS "
+	sql += b.dialect.Quote(table.Name) + " (\n"
+
+	pkList := table.PrimaryKeys
+
+	for _, col := range table.Columns {
+		if col.IsPrimaryKey && len(pkList) == 1 {
+			sql += col.String(b.dialect)
+		} else {
+			sql += col.StringNoPk(b.dialect)
+		}
+		sql = strings.TrimSpace(sql)
+		sql += "\n, "
+	}
+
+	if len(pkList) > 1 {
+		sql += "PRIMARY KEY ( "
+		sql += b.dialect.Quote(strings.Join(pkList, b.dialect.Quote(",")))
+		sql += " ), "
+	}
+
+	sql = sql[:len(sql)-2] + ")"
+	sql += ";"
+	return sql
+}
+
+func (db *BaseDialect) AddColumnSql(tableName string, col *Column) string {
+	return fmt.Sprintf("alter table %s ADD COLUMN %s", tableName, col.StringNoPk(db.dialect))
+}
+
+func (db *BaseDialect) CreateIndexSql(tableName string, index *Index) string {
+	quote := db.dialect.Quote
+	var unique string
+	var idxName string
+	if index.Type == UniqueIndex {
+		unique = " UNIQUE"
+		idxName = fmt.Sprintf("UQE_%v_%v", tableName, index.Name)
+	} else {
+		idxName = fmt.Sprintf("IDX_%v_%v", tableName, index.Name)
+	}
+
+	return fmt.Sprintf("CREATE%s INDEX %v ON %v (%v);", unique,
+		quote(idxName), quote(tableName),
+		quote(strings.Join(index.Cols, quote(","))))
+}

pkg/services/sqlstore/migrator/migrator.go

@@ -0,0 +1,158 @@
+package migrator
+
+import (
+	"time"
+
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/go-xorm/xorm"
+	_ "github.com/lib/pq"
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/torkelo/grafana-pro/pkg/log"
+)
+
+type Migrator struct {
+	LogLevel log.LogLevel
+
+	x          *xorm.Engine
+	dialect    Dialect
+	migrations []Migration
+}
+
+type MigrationLog struct {
+	Id          int64
+	MigrationId string
+	Sql         string
+	Success     bool
+	Error       string
+	Timestamp   time.Time
+}
+
+func NewMigrator(engine *xorm.Engine) *Migrator {
+	mg := &Migrator{}
+	mg.x = engine
+	mg.LogLevel = log.WARN
+	mg.migrations = make([]Migration, 0)
+	mg.dialect = NewDialect(mg.x.DriverName())
+	return mg
+}
+
+func (mg *Migrator) AddMigration(id string, m Migration) {
+	m.SetId(id)
+	mg.migrations = append(mg.migrations, m)
+}
+
+func (mg *Migrator) GetMigrationLog() (map[string]MigrationLog, error) {
+	logMap := make(map[string]MigrationLog)
+	logItems := make([]MigrationLog, 0)
+
+	exists, err := mg.x.IsTableExist(new(MigrationLog))
+	if err != nil {
+		return nil, err
+	}
+
+	if !exists {
+		return logMap, nil
+	}
+
+	if err = mg.x.Find(&logItems); err != nil {
+		return nil, err
+	}
+
+	for _, logItem := range logItems {
+		if !logItem.Success {
+			continue
+		}
+		logMap[logItem.MigrationId] = logItem
+	}
+
+	return logMap, nil
+}
+
+func (mg *Migrator) Start() error {
+	if mg.LogLevel <= log.INFO {
+		log.Info("Migrator:: Starting DB migration")
+	}
+
+	logMap, err := mg.GetMigrationLog()
+	if err != nil {
+		return err
+	}
+
+	for _, m := range mg.migrations {
+		_, exists := logMap[m.Id()]
+		if exists {
+			if mg.LogLevel <= log.DEBUG {
+				log.Debug("Migrator:: Skipping migration: %v, Already executed", m.Id())
+			}
+			continue
+		}
+
+		sql := m.Sql(mg.dialect)
+
+		record := MigrationLog{
+			MigrationId: m.Id(),
+			Sql:         sql,
+			Timestamp:   time.Now(),
+		}
+
+		if mg.LogLevel <= log.DEBUG {
+			log.Debug("Migrator: Executing SQL: \n %v \n", sql)
+		}
+
+		if err := mg.exec(m); err != nil {
+			record.Error = err.Error()
+			mg.x.Insert(&record)
+			return err
+		} else {
+			record.Success = true
+			mg.x.Insert(&record)
+		}
+	}
+
+	return nil
+}
+
+func (mg *Migrator) exec(m Migration) error {
+	if mg.LogLevel <= log.INFO {
+		log.Info("Migrator::exec migration id: %v", m.Id())
+	}
+
+	err := mg.inTransaction(func(sess *xorm.Session) error {
+		_, err := sess.Exec(m.Sql(mg.dialect))
+		if err != nil {
+			log.Error(3, "Migrator::exec FAILED migration id: %v, err: %v", m.Id(), err)
+			return err
+		}
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type dbTransactionFunc func(sess *xorm.Session) error
+
+func (mg *Migrator) inTransaction(callback dbTransactionFunc) error {
+	var err error
+
+	sess := mg.x.NewSession()
+	defer sess.Close()
+
+	if err = sess.Begin(); err != nil {
+		return err
+	}
+
+	err = callback(sess)
+
+	if err != nil {
+		sess.Rollback()
+		return err
+	} else if err = sess.Commit(); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/services/sqlstore/migrator/mysql_dialect.go

@@ -0,0 +1,75 @@
+package migrator
+
+import "strconv"
+
+type Mysql struct {
+	BaseDialect
+}
+
+func NewMysqlDialect() *Mysql {
+	d := Mysql{}
+	d.BaseDialect.dialect = &d
+	d.BaseDialect.driverName = MYSQL
+	return &d
+}
+
+func (db *Mysql) Quote(name string) string {
+	return "`" + name + "`"
+}
+
+func (db *Mysql) QuoteStr() string {
+	return "`"
+}
+
+func (db *Mysql) AutoIncrStr() string {
+	return "AUTO_INCREMENT"
+}
+
+func (db *Mysql) SqlType(c *Column) string {
+	var res string
+	switch c.Type {
+	case DB_Bool:
+		res = DB_TinyInt
+		c.Length = 1
+	case DB_Serial:
+		c.IsAutoIncrement = true
+		c.IsPrimaryKey = true
+		c.Nullable = false
+		res = DB_Int
+	case DB_BigSerial:
+		c.IsAutoIncrement = true
+		c.IsPrimaryKey = true
+		c.Nullable = false
+		res = DB_BigInt
+	case DB_Bytea:
+		res = DB_Blob
+	case DB_TimeStampz:
+		res = DB_Char
+		c.Length = 64
+	case DB_NVarchar:
+		res = DB_Varchar
+	default:
+		res = c.Type
+	}
+
+	var hasLen1 bool = (c.Length > 0)
+	var hasLen2 bool = (c.Length2 > 0)
+
+	if res == DB_BigInt && !hasLen1 && !hasLen2 {
+		c.Length = 20
+		hasLen1 = true
+	}
+
+	if hasLen2 {
+		res += "(" + strconv.Itoa(c.Length) + "," + strconv.Itoa(c.Length2) + ")"
+	} else if hasLen1 {
+		res += "(" + strconv.Itoa(c.Length) + ")"
+	}
+	return res
+}
+
+func (db *Mysql) TableCheckSql(tableName string) (string, []interface{}) {
+	args := []interface{}{"grafana", tableName}
+	sql := "SELECT `TABLE_NAME` from `INFORMATION_SCHEMA`.`TABLES` WHERE `TABLE_SCHEMA`=? and `TABLE_NAME`=?"
+	return sql, args
+}

pkg/services/sqlstore/migrator/postgres_dialect.go

@@ -0,0 +1,82 @@
+package migrator
+
+import "strconv"
+
+type Postgres struct {
+	BaseDialect
+}
+
+func NewPostgresDialect() *Postgres {
+	d := Postgres{}
+	d.BaseDialect.dialect = &d
+	d.BaseDialect.driverName = POSTGRES
+	return &d
+}
+
+func (db *Postgres) Quote(name string) string {
+	return "\"" + name + "\""
+}
+
+func (db *Postgres) QuoteStr() string {
+	return "\""
+}
+
+func (db *Postgres) AutoIncrStr() string {
+	return ""
+}
+
+func (db *Postgres) SqlType(c *Column) string {
+	var res string
+	switch t := c.Type; t {
+	case DB_TinyInt:
+		res = DB_SmallInt
+		return res
+	case DB_MediumInt, DB_Int, DB_Integer:
+		if c.IsAutoIncrement {
+			return DB_Serial
+		}
+		return DB_Integer
+	case DB_Serial, DB_BigSerial:
+		c.IsAutoIncrement = true
+		c.Nullable = false
+		res = t
+	case DB_Binary, DB_VarBinary:
+		return DB_Bytea
+	case DB_DateTime:
+		res = DB_TimeStamp
+	case DB_TimeStampz:
+		return "timestamp with time zone"
+	case DB_Float:
+		res = DB_Real
+	case DB_TinyText, DB_MediumText, DB_LongText:
+		res = DB_Text
+	case DB_NVarchar:
+		res = DB_Varchar
+	case DB_Uuid:
+		res = DB_Uuid
+	case DB_Blob, DB_TinyBlob, DB_MediumBlob, DB_LongBlob:
+		return DB_Bytea
+	case DB_Double:
+		return "DOUBLE PRECISION"
+	default:
+		if c.IsAutoIncrement {
+			return DB_Serial
+		}
+		res = t
+	}
+
+	var hasLen1 bool = (c.Length > 0)
+	var hasLen2 bool = (c.Length2 > 0)
+	if hasLen2 {
+		res += "(" + strconv.Itoa(c.Length) + "," + strconv.Itoa(c.Length2) + ")"
+	} else if hasLen1 {
+		res += "(" + strconv.Itoa(c.Length) + ")"
+	}
+	return res
+}
+
+func (db *Postgres) TableCheckSql(tableName string) (string, []interface{}) {
+	args := []interface{}{"grafana", tableName}
+	sql := "SELECT `TABLE_NAME` from `INFORMATION_SCHEMA`.`TABLES` WHERE `TABLE_SCHEMA`=? and `TABLE_NAME`=?"
+	return sql, args
+}

pkg/services/sqlstore/migrator/sqlite_dialect.go

@@ -0,0 +1,55 @@
+package migrator
+
+type Sqlite3 struct {
+	BaseDialect
+}
+
+func NewSqlite3Dialect() *Sqlite3 {
+	d := Sqlite3{}
+	d.BaseDialect.dialect = &d
+	d.BaseDialect.driverName = SQLITE
+	return &d
+}
+
+func (db *Sqlite3) Quote(name string) string {
+	return "`" + name + "`"
+}
+
+func (db *Sqlite3) QuoteStr() string {
+	return "`"
+}
+
+func (db *Sqlite3) AutoIncrStr() string {
+	return "AUTOINCREMENT"
+}
+
+func (db *Sqlite3) SqlType(c *Column) string {
+	switch c.Type {
+	case DB_Date, DB_DateTime, DB_TimeStamp, DB_Time:
+		return DB_DateTime
+	case DB_TimeStampz:
+		return DB_Text
+	case DB_Char, DB_Varchar, DB_NVarchar, DB_TinyText, DB_Text, DB_MediumText, DB_LongText:
+		return DB_Text
+	case DB_Bit, DB_TinyInt, DB_SmallInt, DB_MediumInt, DB_Int, DB_Integer, DB_BigInt, DB_Bool:
+		return DB_Integer
+	case DB_Float, DB_Double, DB_Real:
+		return DB_Real
+	case DB_Decimal, DB_Numeric:
+		return DB_Numeric
+	case DB_TinyBlob, DB_Blob, DB_MediumBlob, DB_LongBlob, DB_Bytea, DB_Binary, DB_VarBinary:
+		return DB_Blob
+	case DB_Serial, DB_BigSerial:
+		c.IsPrimaryKey = true
+		c.IsAutoIncrement = true
+		c.Nullable = false
+		return DB_Integer
+	default:
+		return c.Type
+	}
+}
+
+func (db *Sqlite3) TableCheckSql(tableName string) (string, []interface{}) {
+	args := []interface{}{tableName}
+	return "SELECT name FROM sqlite_master WHERE type='table' and name = ?", args
+}

pkg/services/sqlstore/migrator/types.go

@@ -0,0 +1,86 @@
+package migrator
+
+const (
+	POSTGRES = "postgres"
+	SQLITE   = "sqlite3"
+	MYSQL    = "mysql"
+)
+
+type Migration interface {
+	Sql(dialect Dialect) string
+	Id() string
+	SetId(string)
+}
+
+type SQLType string
+
+type ColumnType string
+
+const (
+	DB_TYPE_STRING ColumnType = "String"
+)
+
+type Table struct {
+	Name        string
+	Columns     []*Column
+	PrimaryKeys []string
+}
+
+const (
+	IndexType = iota + 1
+	UniqueIndex
+)
+
+type Index struct {
+	Name string
+	Type int
+	Cols []string
+}
+
+var (
+	DB_Bit       = "BIT"
+	DB_TinyInt   = "TINYINT"
+	DB_SmallInt  = "SMALLINT"
+	DB_MediumInt = "MEDIUMINT"
+	DB_Int       = "INT"
+	DB_Integer   = "INTEGER"
+	DB_BigInt    = "BIGINT"
+
+	DB_Enum = "ENUM"
+	DB_Set  = "SET"
+
+	DB_Char       = "CHAR"
+	DB_Varchar    = "VARCHAR"
+	DB_NVarchar   = "NVARCHAR"
+	DB_TinyText   = "TINYTEXT"
+	DB_Text       = "TEXT"
+	DB_MediumText = "MEDIUMTEXT"
+	DB_LongText   = "LONGTEXT"
+	DB_Uuid       = "UUID"
+
+	DB_Date       = "DATE"
+	DB_DateTime   = "DATETIME"
+	DB_Time       = "TIME"
+	DB_TimeStamp  = "TIMESTAMP"
+	DB_TimeStampz = "TIMESTAMPZ"
+
+	DB_Decimal = "DECIMAL"
+	DB_Numeric = "NUMERIC"
+
+	DB_Real   = "REAL"
+	DB_Float  = "FLOAT"
+	DB_Double = "DOUBLE"
+
+	DB_Binary     = "BINARY"
+	DB_VarBinary  = "VARBINARY"
+	DB_TinyBlob   = "TINYBLOB"
+	DB_Blob       = "BLOB"
+	DB_MediumBlob = "MEDIUMBLOB"
+	DB_LongBlob   = "LONGBLOB"
+	DB_Bytea      = "BYTEA"
+
+	DB_Bool = "BOOL"
+
+	DB_Serial    = "SERIAL"
+	DB_BigSerial = "BIGSERIAL"
+)