Merge pull request #14577 from marefr/14351_db_migration

fix only create/drop database indices if not exists/exists

pkg/services/sqlstore/migrator/conditions.go

@@ -2,12 +2,47 @@ package migrator
 
 type MigrationCondition interface {
 	Sql(dialect Dialect) (string, []interface{})
+	IsFulfilled(results []map[string][]byte) bool
 }
 
-type IfTableExistsCondition struct {
+type ExistsMigrationCondition struct{}
+
+func (c *ExistsMigrationCondition) IsFulfilled(results []map[string][]byte) bool {
+	return len(results) >= 1
+}
+
+type NotExistsMigrationCondition struct{}
+
+func (c *NotExistsMigrationCondition) IsFulfilled(results []map[string][]byte) bool {
+	return len(results) == 0
+}
+
+type IfIndexExistsCondition struct {
+	ExistsMigrationCondition
 	TableName string
+	IndexName string
 }
 
-func (c *IfTableExistsCondition) Sql(dialect Dialect) (string, []interface{}) {
+func (c *IfIndexExistsCondition) Sql(dialect Dialect) (string, []interface{}) {
-	return dialect.TableCheckSql(c.TableName)
+	return dialect.IndexCheckSql(c.TableName, c.IndexName)
+}
+
+type IfIndexNotExistsCondition struct {
+	NotExistsMigrationCondition
+	TableName string
+	IndexName string
+}
+
+func (c *IfIndexNotExistsCondition) Sql(dialect Dialect) (string, []interface{}) {
+	return dialect.IndexCheckSql(c.TableName, c.IndexName)
+}
+
+type IfColumnNotExistsCondition struct {
+	NotExistsMigrationCondition
+	TableName  string
+	ColumnName string
+}
+
+func (c *IfColumnNotExistsCondition) Sql(dialect Dialect) (string, []interface{}) {
+	return dialect.ColumnCheckSql(c.TableName, c.ColumnName)
 }

pkg/services/sqlstore/migrator/dialect.go

@@ -29,10 +29,12 @@ type Dialect interface {
 	DropTable(tableName string) string
 	DropIndexSql(tableName string, index *Index) string
 
-	TableCheckSql(tableName string) (string, []interface{})
 	RenameTable(oldName string, newName string) string
 	UpdateTableSql(tableName string, columns []*Column) string
 
+	IndexCheckSql(tableName, indexName string) (string, []interface{})
+	ColumnCheckSql(tableName, columnName string) (string, []interface{})
+
 	ColString(*Column) string
 	ColStringNoPk(*Column) string
 
@@ -182,6 +184,10 @@ func (db *BaseDialect) RenameTable(oldName string, newName string) string {
 	return fmt.Sprintf("ALTER TABLE %s RENAME TO %s", quote(oldName), quote(newName))
 }
 
+func (db *BaseDialect) ColumnCheckSql(tableName, columnName string) (string, []interface{}) {
+	return "", nil
+}
+
 func (db *BaseDialect) DropIndexSql(tableName string, index *Index) string {
 	quote := db.dialect.Quote
 	name := index.XName(tableName)

pkg/services/sqlstore/migrator/migrations.go

@@ -85,7 +85,9 @@ type AddColumnMigration struct {
 }
 
 func NewAddColumnMigration(table Table, col *Column) *AddColumnMigration {
-	return &AddColumnMigration{tableName: table.Name, column: col}
+	m := &AddColumnMigration{tableName: table.Name, column: col}
+	m.Condition = &IfColumnNotExistsCondition{TableName: table.Name, ColumnName: col.Name}
+	return m
 }
 
 func (m *AddColumnMigration) Table(tableName string) *AddColumnMigration {
@@ -109,7 +111,9 @@ type AddIndexMigration struct {
 }
 
 func NewAddIndexMigration(table Table, index *Index) *AddIndexMigration {
-	return &AddIndexMigration{tableName: table.Name, index: index}
+	m := &AddIndexMigration{tableName: table.Name, index: index}
+	m.Condition = &IfIndexNotExistsCondition{TableName: table.Name, IndexName: index.XName(table.Name)}
+	return m
 }
 
 func (m *AddIndexMigration) Table(tableName string) *AddIndexMigration {
@@ -128,7 +132,9 @@ type DropIndexMigration struct {
 }
 
 func NewDropIndexMigration(table Table, index *Index) *DropIndexMigration {
-	return &DropIndexMigration{tableName: table.Name, index: index}
+	m := &DropIndexMigration{tableName: table.Name, index: index}
+	m.Condition = &IfIndexExistsCondition{TableName: table.Name, IndexName: index.XName(table.Name)}
+	return m
 }
 
 func (m *DropIndexMigration) Sql(dialect Dialect) string {
@@ -179,11 +185,6 @@ func NewRenameTableMigration(oldName string, newName string) *RenameTableMigrati
 	return &RenameTableMigration{oldName: oldName, newName: newName}
 }
 
-func (m *RenameTableMigration) IfTableExists(tableName string) *RenameTableMigration {
-	m.Condition = &IfTableExistsCondition{TableName: tableName}
-	return m
-}
-
 func (m *RenameTableMigration) Rename(oldName string, newName string) *RenameTableMigration {
 	m.oldName = oldName
 	m.newName = newName
@@ -212,11 +213,6 @@ func NewCopyTableDataMigration(targetTable string, sourceTable string, colMap ma
 	return m
 }
 
-func (m *CopyTableDataMigration) IfTableExists(tableName string) *CopyTableDataMigration {
-	m.Condition = &IfTableExistsCondition{TableName: tableName}
-	return m
-}
-
 func (m *CopyTableDataMigration) Sql(d Dialect) string {
 	return d.CopyTableData(m.sourceTable, m.targetTable, m.sourceCols, m.targetCols)
 }

pkg/services/sqlstore/migrator/migrator.go

@@ -94,8 +94,6 @@ func (mg *Migrator) Start() error {
 			Timestamp:   time.Now(),
 		}
 
-		mg.Logger.Debug("Executing", "sql", sql)
-
 		err := mg.inTransaction(func(sess *xorm.Session) error {
 			err := mg.exec(m, sess)
 			if err != nil {
@@ -123,18 +121,30 @@ func (mg *Migrator) exec(m Migration, sess *xorm.Session) error {
 	condition := m.GetCondition()
 	if condition != nil {
 		sql, args := condition.Sql(mg.Dialect)
-		results, err := sess.SQL(sql).Query(args...)
+
-		if err != nil || len(results) == 0 {
+		if sql != "" {
-			mg.Logger.Debug("Skipping migration condition not fulfilled", "id", m.Id())
+			mg.Logger.Debug("Executing migration condition sql", "id", m.Id(), "sql", sql, "args", args)
-			return sess.Rollback()
+			results, err := sess.SQL(sql, args...).Query()
+			if err != nil {
+				mg.Logger.Error("Executing migration condition failed", "id", m.Id(), "error", err)
+				return err
+			}
+
+			if !condition.IsFulfilled(results) {
+				mg.Logger.Warn("Skipping migration: Already executed, but not recorded in migration log", "id", m.Id())
+				return nil
+			}
 		}
 	}
 
 	var err error
 	if codeMigration, ok := m.(CodeMigration); ok {
+		mg.Logger.Debug("Executing code migration", "id", m.Id())
 		err = codeMigration.Exec(sess, mg)
 	} else {
-		_, err = sess.Exec(m.Sql(mg.Dialect))
+		sql := m.Sql(mg.Dialect)
+		mg.Logger.Debug("Executing sql migration", "id", m.Id(), "sql", sql)
+		_, err = sess.Exec(sql)
 	}
 
 	if err != nil {

pkg/services/sqlstore/migrator/mysql_dialect.go

@@ -90,12 +90,6 @@ func (db *Mysql) SqlType(c *Column) string {
 	return res
 }
 
-func (db *Mysql) TableCheckSql(tableName string) (string, []interface{}) {
-	args := []interface{}{"grafana", tableName}
-	sql := "SELECT `TABLE_NAME` from `INFORMATION_SCHEMA`.`TABLES` WHERE `TABLE_SCHEMA`=? and `TABLE_NAME`=?"
-	return sql, args
-}
-
 func (db *Mysql) UpdateTableSql(tableName string, columns []*Column) string {
 	var statements = []string{}
 
@@ -108,6 +102,18 @@ func (db *Mysql) UpdateTableSql(tableName string, columns []*Column) string {
 	return "ALTER TABLE " + db.Quote(tableName) + " " + strings.Join(statements, ", ") + ";"
 }
 
+func (db *Mysql) IndexCheckSql(tableName, indexName string) (string, []interface{}) {
+	args := []interface{}{tableName, indexName}
+	sql := "SELECT 1 FROM " + db.Quote("INFORMATION_SCHEMA") + "." + db.Quote("STATISTICS") + " WHERE " + db.Quote("TABLE_SCHEMA") + " = DATABASE() AND " + db.Quote("TABLE_NAME") + "=? AND " + db.Quote("INDEX_NAME") + "=?"
+	return sql, args
+}
+
+func (db *Mysql) ColumnCheckSql(tableName, columnName string) (string, []interface{}) {
+	args := []interface{}{tableName, columnName}
+	sql := "SELECT 1 FROM " + db.Quote("INFORMATION_SCHEMA") + "." + db.Quote("COLUMNS") + " WHERE " + db.Quote("TABLE_SCHEMA") + " = DATABASE() AND " + db.Quote("TABLE_NAME") + "=? AND " + db.Quote("COLUMN_NAME") + "=?"
+	return sql, args
+}
+
 func (db *Mysql) CleanDB() error {
 	tables, _ := db.engine.DBMetas()
 	sess := db.engine.NewSession()

pkg/services/sqlstore/migrator/postgres_dialect.go

@@ -101,9 +101,9 @@ func (db *Postgres) SqlType(c *Column) string {
 	return res
 }
 
-func (db *Postgres) TableCheckSql(tableName string) (string, []interface{}) {
+func (db *Postgres) IndexCheckSql(tableName, indexName string) (string, []interface{}) {
-	args := []interface{}{"grafana", tableName}
+	args := []interface{}{tableName, indexName}
-	sql := "SELECT table_name FROM information_schema.tables WHERE table_schema=? and table_name=?"
+	sql := "SELECT 1 FROM " + db.Quote("pg_indexes") + " WHERE" + db.Quote("tablename") + "=? AND " + db.Quote("indexname") + "=?"
 	return sql, args
 }
 

pkg/services/sqlstore/migrator/sqlite_dialect.go

@@ -68,9 +68,10 @@ func (db *Sqlite3) SqlType(c *Column) string {
 	}
 }
 
-func (db *Sqlite3) TableCheckSql(tableName string) (string, []interface{}) {
+func (db *Sqlite3) IndexCheckSql(tableName, indexName string) (string, []interface{}) {
-	args := []interface{}{tableName}
+	args := []interface{}{tableName, indexName}
-	return "SELECT name FROM sqlite_master WHERE type='table' and name = ?", args
+	sql := "SELECT 1 FROM " + db.Quote("sqlite_master") + " WHERE " + db.Quote("type") + "='index' AND " + db.Quote("tbl_name") + "=? AND " + db.Quote("name") + "=?"
+	return sql, args
 }
 
 func (db *Sqlite3) DropIndexSql(tableName string, index *Index) string {