Encryption: Add support for multiple data keys per day (#47765)

* Add database migrations

* Use short uids as data key ids

* Add support for manual data key rotation

* Fix duplicated mutex unlocks

* Fix migration

* Manage current data keys per name

* Adjust key re-encryption and test

* Modify rename column migration for MySQL compatibility

* Refactor secrets manager and data keys cache

* Multiple o11y adjustments

* Fix stats query

* Apply suggestions from code review

Co-authored-by: Tania <yalyna.ts@gmail.com>

* Fix linter

* Docs: Rotate data encryption keys API endpoint

Co-authored-by: Tania <yalyna.ts@gmail.com>

pkg/services/sqlstore/migrations/secrets_mig.go

@@ -1,6 +1,10 @@
 package migrations
 
-import "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
+import (
+	"fmt"
+
+	"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
+)
 
 func addSecretsMigration(mg *migrator.Migrator) {
 	dataKeysV1 := migrator.Table{
@@ -38,4 +42,23 @@ func addSecretsMigration(mg *migrator.Migrator) {
 	}
 
 	mg.AddMigration("create secrets table", migrator.NewAddTableMigration(secretsV1))
+
+	mg.AddMigration("rename data_keys name column to id", migrator.NewRenameColumnMigration(
+		dataKeysV1, "name", "id",
+	))
+
+	mg.AddMigration("add name column into data_keys", migrator.NewAddColumnMigration(
+		dataKeysV1,
+		&migrator.Column{
+			Name:     "name",
+			Type:     migrator.DB_NVarchar,
+			Length:   100,
+			Default:  "''",
+			Nullable: false,
+		},
+	))
+
+	mg.AddMigration("copy data_keys id column values into name", migrator.NewRawSQLMigration(
+		fmt.Sprintf("UPDATE %s SET %s = %s", dataKeysV1.Name, "name", "id"),
+	))
 }

pkg/services/sqlstore/migrator/dialect.go

@@ -37,6 +37,8 @@ type Dialect interface {
 	DropIndexSQL(tableName string, index *Index) string
 
 	RenameTable(oldName string, newName string) string
+	RenameColumn(table Table, oldName, newName string) string
+
 	UpdateTableSQL(tableName string, columns []*Column) string
 
 	IndexCheckSQL(tableName, indexName string) (string, []interface{})
@@ -209,6 +211,11 @@ func (b *BaseDialect) RenameTable(oldName string, newName string) string {
 	return fmt.Sprintf("ALTER TABLE %s RENAME TO %s", quote(oldName), quote(newName))
 }
 
+func (b *BaseDialect) RenameColumn(table Table, oldName, newName string) string {
+	quote := b.dialect.Quote
+	return fmt.Sprintf("ALTER TABLE %s RENAME COLUMN %s TO %s", quote(table.Name), quote(oldName), quote(newName))
+}
+
 func (b *BaseDialect) ColumnCheckSQL(tableName, columnName string) (string, []interface{}) {
 	return "", nil
 }

pkg/services/sqlstore/migrator/migrations.go

@@ -108,6 +108,32 @@ func (m *AddColumnMigration) SQL(dialect Dialect) string {
 	return dialect.AddColumnSQL(m.tableName, m.column)
 }
 
+type RenameColumnMigration struct {
+	MigrationBase
+	table   Table
+	oldName string
+	newName string
+}
+
+func NewRenameColumnMigration(table Table, oldName, newName string) *RenameColumnMigration {
+	return &RenameColumnMigration{table: table, oldName: oldName, newName: newName}
+}
+
+func (m *RenameColumnMigration) Table(table Table) *RenameColumnMigration {
+	m.table = table
+	return m
+}
+
+func (m *RenameColumnMigration) Rename(oldName string, newName string) *RenameColumnMigration {
+	m.oldName = oldName
+	m.newName = newName
+	return m
+}
+
+func (m *RenameColumnMigration) SQL(d Dialect) string {
+	return d.RenameColumn(m.table, m.oldName, m.newName)
+}
+
 type AddIndexMigration struct {
 	MigrationBase
 	tableName string

pkg/services/sqlstore/migrator/mysql_dialect.go

@@ -118,6 +118,20 @@ func (db *MySQLDialect) ColumnCheckSQL(tableName, columnName string) (string, []
 	return sql, args
 }
 
+func (db *MySQLDialect) RenameColumn(table Table, oldName, newName string) string {
+	var colType string
+	for _, col := range table.Columns {
+		if col.Name == oldName {
+			colType = db.SQLType(col)
+			break
+		}
+	}
+
+	quote := db.dialect.Quote
+
+	return fmt.Sprintf("ALTER TABLE %s CHANGE %s %s %s", quote(table.Name), quote(oldName), quote(newName), colType)
+}
+
 func (db *MySQLDialect) CleanDB() error {
 	tables, err := db.engine.DBMetas()
 	if err != nil {

pkg/services/sqlstore/stats.go

@@ -103,6 +103,7 @@ func (ss *SQLStore) GetSystemStats(ctx context.Context, query *models.GetSystemS
 		sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("library_element")+` WHERE kind = ?) AS library_panels,`, models.PanelElement)
 		sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("library_element")+` WHERE kind = ?) AS library_variables,`, models.VariableElement)
 		sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("data_keys") + `) AS data_keys,`)
+		sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("data_keys") + `WHERE active = true) AS active_data_keys,`)
 
 		sb.Write(ss.roleCounterSQL(ctx))