Encryption: Refine secrets-related commands (#45201)

* CLI: Adjust 're-encrypt-data-keys' command

* CLI: Adjust 're-encrypt' command

* Multiple improvements on re-encrypt secrets migration

* Another bunch of code improvements

* Lint fixes

pkg/cmd/grafana-cli/commands/secretsmigrations/reencrypt_deks.go

@@ -3,7 +3,6 @@ package secretsmigrations
 import (
 	"context"
 
-	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/runner"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"

pkg/cmd/grafana-cli/commands/secretsmigrations/reencrypt_secrets.go

@@ -4,9 +4,9 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 
-	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/runner"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
@@ -17,69 +17,123 @@ import (
 	"xorm.io/xorm"
 )
 
-func (s simpleSecret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm.Session) error {
+func (s simpleSecret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm.Session) {
 	var rows []struct {
 		Id     int
-		Secret string
+		Secret []byte
 	}
 
 	if err := sess.Table(s.tableName).Select(fmt.Sprintf("id, %s as secret", s.columnName)).Find(&rows); err != nil {
-		return err
+		logger.Warn("Could not find any secret to re-encrypt", "table", s.tableName)
+		return
 	}
 
+	var anyFailure bool
+
 	for _, row := range rows {
 		if len(row.Secret) == 0 {
 			continue
 		}
 
-		var (
-			err     error
-			decoded = []byte(row.Secret)
-		)
-
-		if s.isBase64Encoded {
-			decoded, err = base64.StdEncoding.DecodeString(row.Secret)
-			if err != nil {
-				return err
-			}
-		}
-
-		decrypted, err := secretsSrv.Decrypt(context.Background(), decoded)
+		decrypted, err := secretsSrv.Decrypt(context.Background(), row.Secret)
 		if err != nil {
-			return err
+			anyFailure = true
+			logger.Warn("Could not decrypt secret while re-encrypting it", "table", s.tableName, "id", row.Id, "error", err)
+			continue
 		}
 
 		encrypted, err := secretsSrv.EncryptWithDBSession(context.Background(), decrypted, secrets.WithoutScope(), sess)
 		if err != nil {
-			return err
+			anyFailure = true
+			logger.Warn("Could not encrypt secret while re-encrypting it", "table", s.tableName, "id", row.Id, "error", err)
+			continue
 		}
 
-		encoded := string(encrypted)
-		if s.isBase64Encoded {
-			encoded = base64.StdEncoding.EncodeToString(encrypted)
-		}
-
-		updateSQL := fmt.Sprintf("UPDATE %s SET %s = ? WHERE id = ?", s.tableName, s.columnName)
-		if _, err := sess.Exec(updateSQL, encoded, row.Id); err != nil {
-			return err
+		updateSQL := fmt.Sprintf("UPDATE %s SET %s = ?, updated = ? WHERE id = ?", s.tableName, s.columnName)
+		if _, err = sess.Exec(updateSQL, encrypted, nowInUTC(), row.Id); err != nil {
+			anyFailure = true
+			logger.Warn("Could not update secret while re-encrypting it", "table", s.tableName, "id", row.Id, "error", err)
+			continue
 		}
 	}
 
-	logger.Infof("Column %s from %s has been re-encrypted successfully\n", s.columnName, s.tableName)
-
-	return nil
+	if anyFailure {
+		logger.Warn(fmt.Sprintf("Column %s from %s has been re-encrypted with errors", s.columnName, s.tableName))
+	} else {
+		logger.Info(fmt.Sprintf("Column %s from %s has been re-encrypted successfully", s.columnName, s.tableName))
+	}
 }
 
-func (s jsonSecret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm.Session) error {
+func (s b64Secret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm.Session) {
+	var rows []struct {
+		Id     int
+		Secret string
+	}
+
+	if err := sess.Table(s.tableName).Select(fmt.Sprintf("id, %s as secret", s.columnName)).Find(&rows); err != nil {
+		logger.Warn("Could not find any secret to re-encrypt", "table", s.tableName)
+		return
+	}
+
+	var anyFailure bool
+
+	for _, row := range rows {
+		if len(row.Secret) == 0 {
+			continue
+		}
+
+		decoded, err := base64.StdEncoding.DecodeString(row.Secret)
+		if err != nil {
+			anyFailure = true
+			logger.Warn("Could not decode base64-encoded secret while re-encrypting it", "table", s.tableName, "id", row.Id, "error", err)
+			continue
+		}
+
+		decrypted, err := secretsSrv.Decrypt(context.Background(), decoded)
+		if err != nil {
+			anyFailure = true
+			logger.Warn("Could not decrypt secret while re-encrypting it", "table", s.tableName, "id", row.Id, "error", err)
+			continue
+		}
+
+		encrypted, err := secretsSrv.EncryptWithDBSession(context.Background(), decrypted, secrets.WithoutScope(), sess)
+		if err != nil {
+			anyFailure = true
+			logger.Warn("Could not encrypt secret while re-encrypting it", "table", s.tableName, "id", row.Id, "error", err)
+			continue
+		}
+
+		encoded := base64.StdEncoding.EncodeToString(encrypted)
+		updateSQL := fmt.Sprintf("UPDATE %s SET %s = ? WHERE id = ?", s.tableName, s.columnName)
+		_, err = sess.Exec(updateSQL, encoded, row.Id)
+
+		if err != nil {
+			anyFailure = true
+			logger.Warn("Could not update secret while re-encrypting it", "table", s.tableName, "id", row.Id, "error", err)
+			continue
+		}
+	}
+
+	if anyFailure {
+		logger.Warn(fmt.Sprintf("Column %s from %s has been re-encrypted with errors", s.columnName, s.tableName))
+	} else {
+		logger.Info(fmt.Sprintf("Column %s from %s has been re-encrypted successfully", s.columnName, s.tableName))
+	}
+}
+
+func (s jsonSecret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm.Session) {
 	var rows []struct {
 		Id             int
 		SecureJsonData map[string][]byte
 	}
 
 	if err := sess.Table(s.tableName).Cols("id", "secure_json_data").Find(&rows); err != nil {
-		return err
+		logger.Warn("Could not find any secret to re-encrypt", "table", s.tableName)
+		return
 	}
 
+	var anyFailure bool
+
 	for _, row := range rows {
 		if len(row.SecureJsonData) == 0 {
 			continue
@@ -87,29 +141,38 @@ func (s jsonSecret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm.Ses
 
 		decrypted, err := secretsSrv.DecryptJsonData(context.Background(), row.SecureJsonData)
 		if err != nil {
-			return err
+			anyFailure = true
+			logger.Warn("Could not decrypt secrets while re-encrypting them", "table", s.tableName, "id", row.Id, "error", err)
+			continue
 		}
 
-		var toUpdate struct {
+		toUpdate := struct {
 			SecureJsonData map[string][]byte
-		}
+			Updated        string
+		}{Updated: nowInUTC()}
 
 		toUpdate.SecureJsonData, err = secretsSrv.EncryptJsonDataWithDBSession(context.Background(), decrypted, secrets.WithoutScope(), sess)
 		if err != nil {
-			return err
+			anyFailure = true
+			logger.Warn("Could not re-encrypt secrets", "table", s.tableName, "id", row.Id, "error", err)
+			continue
 		}
 
 		if _, err := sess.Table(s.tableName).Where("id = ?", row.Id).Update(toUpdate); err != nil {
-			return err
+			anyFailure = true
+			logger.Warn("Could not update secrets while re-encrypting them", "table", s.tableName, "id", row.Id, "error", err)
+			continue
 		}
 	}
 
-	logger.Infof("Secure json data from %s has been re-encrypted successfully\n", s.tableName)
-
-	return nil
+	if anyFailure {
+		logger.Warn(fmt.Sprintf("Secure json data secrets from %s have been re-encrypted with errors", s.tableName))
+	} else {
+		logger.Info(fmt.Sprintf("Secure json data secrets from %s have been re-encrypted successfully", s.tableName))
+	}
 }
 
-func (s alertingSecret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm.Session) error {
+func (s alertingSecret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm.Session) {
 	var results []struct {
 		Id                        int
 		AlertmanagerConfiguration string
@@ -117,14 +180,19 @@ func (s alertingSecret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm
 
 	selectSQL := "SELECT id, alertmanager_configuration FROM alert_configuration"
 	if err := sess.SQL(selectSQL).Find(&results); err != nil {
-		return err
+		logger.Warn("Could not find any alert_configuration secret to re-encrypt")
+		return
 	}
 
+	var anyFailure bool
+
 	for _, result := range results {
 		result := result
 		postableUserConfig, err := notifier.Load([]byte(result.AlertmanagerConfiguration))
 		if err != nil {
-			return err
+			anyFailure = true
+			logger.Warn("Could not load alert_configuration while re-encrypting it", "id", result.Id, "error", err)
+			continue
 		}
 
 		for _, receiver := range postableUserConfig.AlertmanagerConfig.Receivers {
@@ -132,17 +200,23 @@ func (s alertingSecret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm
 				for k, v := range gmr.SecureSettings {
 					decoded, err := base64.StdEncoding.DecodeString(v)
 					if err != nil {
-						return err
+						anyFailure = true
+						logger.Warn("Could not decode base64-encoded alert_configuration secret", "id", result.Id, "key", k, "error", err)
+						continue
 					}
 
 					decrypted, err := secretsSrv.Decrypt(context.Background(), decoded)
 					if err != nil {
-						return err
+						anyFailure = true
+						logger.Warn("Could not decrypt alert_configuration secret", "id", result.Id, "key", k, "error", err)
+						continue
 					}
 
 					reencrypted, err := secretsSrv.EncryptWithDBSession(context.Background(), decrypted, secrets.WithoutScope(), sess)
 					if err != nil {
-						return err
+						anyFailure = true
+						logger.Warn("Could not re-encrypt alert_configuration secret", "id", result.Id, "key", k, "error", err)
+						continue
 					}
 
 					gmr.SecureSettings[k] = base64.StdEncoding.EncodeToString(reencrypted)
@@ -152,18 +226,24 @@ func (s alertingSecret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm
 
 		marshalled, err := json.Marshal(postableUserConfig)
 		if err != nil {
-			return err
+			anyFailure = true
+			logger.Warn("Could not marshal alert_configuration while re-encrypting it", "id", result.Id, "error", err)
+			continue
 		}
 
 		result.AlertmanagerConfiguration = string(marshalled)
 		if _, err := sess.Table("alert_configuration").Where("id = ?", result.Id).Update(&result); err != nil {
-			return err
+			anyFailure = true
+			logger.Warn("Could not update alert_configuration secret while re-encrypting it", "id", result.Id, "error", err)
+			continue
 		}
 	}
 
-	logger.Info("Alerting secrets has been re-encrypted successfully\n")
-
-	return nil
+	if anyFailure {
+		logger.Warn("Alerting configuration secrets have been re-encrypted with errors")
+	} else {
+		logger.Info("Alerting configuration secrets have been re-encrypted successfully")
+	}
 }
 
 func ReEncryptSecrets(_ utils.CommandLine, runner runner.Runner) error {
@@ -173,22 +253,27 @@ func ReEncryptSecrets(_ utils.CommandLine, runner runner.Runner) error {
 	}
 
 	toMigrate := []interface {
-		reencrypt(*manager.SecretsService, *xorm.Session) error
+		reencrypt(*manager.SecretsService, *xorm.Session)
 	}{
-		simpleSecret{tableName: "dashboard_snapshot", columnName: "dashboard_encrypted", isBase64Encoded: false},
-		simpleSecret{tableName: "user_auth", columnName: "o_auth_access_token", isBase64Encoded: true},
-		simpleSecret{tableName: "user_auth", columnName: "o_auth_refresh_token", isBase64Encoded: true},
-		simpleSecret{tableName: "user_auth", columnName: "o_auth_token_type", isBase64Encoded: true},
+		simpleSecret{tableName: "dashboard_snapshot", columnName: "dashboard_encrypted"},
+		b64Secret{simpleSecret{tableName: "user_auth", columnName: "o_auth_access_token"}},
+		b64Secret{simpleSecret{tableName: "user_auth", columnName: "o_auth_refresh_token"}},
+		b64Secret{simpleSecret{tableName: "user_auth", columnName: "o_auth_token_type"}},
 		jsonSecret{tableName: "data_source"},
 		jsonSecret{tableName: "plugin_setting"},
 		alertingSecret{},
 	}
 
-	return runner.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
-		for _, m := range toMigrate {
-			if err := m.reencrypt(runner.SecretsService, sess.Session); err != nil {
-				return err
+	return runner.SQLStore.WithTransactionalDbSession(context.Background(), func(sess *sqlstore.DBSession) (err error) {
+		defer func() {
+			if r := recover(); r != nil {
+				err = errors.New(fmt.Sprint(r))
+				logger.Error("Secrets re-encryption failed, rolling back transaction...", "error", err)
 			}
+		}()
+
+		for _, m := range toMigrate {
+			m.reencrypt(runner.SecretsService, sess.Session)
 		}
 
 		return nil

pkg/cmd/grafana-cli/commands/secretsmigrations/rollback_secrets.go

@@ -4,14 +4,13 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 
-	"github.com/grafana/grafana/pkg/services/featuremgmt"
-
-	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/runner"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 	"github.com/grafana/grafana/pkg/services/encryption"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
 	"github.com/grafana/grafana/pkg/services/secrets/manager"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
@@ -23,14 +22,15 @@ func (s simpleSecret) rollback(
 	encryptionSrv encryption.Internal,
 	sess *xorm.Session,
 	secretKey string,
-) error {
+) (anyFailure bool) {
 	var rows []struct {
 		Id     int
-		Secret string
+		Secret []byte
 	}
 
 	if err := sess.Table(s.tableName).Select(fmt.Sprintf("id, %s as secret", s.columnName)).Find(&rows); err != nil {
-		return err
+		logger.Warn("Could not find any secret to roll back", "table", s.tableName)
+		return true
 	}
 
 	for _, row := range rows {
@@ -38,42 +38,95 @@ func (s simpleSecret) rollback(
 			continue
 		}
 
-		var (
-			err     error
-			decoded = []byte(row.Secret)
-		)
-
-		if s.isBase64Encoded {
-			decoded, err = base64.StdEncoding.DecodeString(row.Secret)
-			if err != nil {
-				return err
-			}
-		}
-
-		decrypted, err := secretsSrv.Decrypt(context.Background(), decoded)
+		decrypted, err := secretsSrv.Decrypt(context.Background(), row.Secret)
 		if err != nil {
-			return err
+			anyFailure = true
+			logger.Warn("Could not decrypt secret while rolling it back", "table", s.tableName, "id", row.Id, "error", err)
+			continue
 		}
 
 		encrypted, err := encryptionSrv.Encrypt(context.Background(), decrypted, secretKey)
 		if err != nil {
-			return err
+			anyFailure = true
+			logger.Warn("Could not encrypt secret while rolling it back", "table", s.tableName, "id", row.Id, "error", err)
+			continue
 		}
 
-		encoded := string(encrypted)
-		if s.isBase64Encoded {
-			encoded = base64.StdEncoding.EncodeToString(encrypted)
-		}
-
-		updateSQL := fmt.Sprintf("UPDATE %s SET %s = ? WHERE id = ?", s.tableName, s.columnName)
-		if _, err := sess.Exec(updateSQL, encoded, row.Id); err != nil {
-			return err
+		updateSQL := fmt.Sprintf("UPDATE %s SET %s = ?, updated = ? WHERE id = ?", s.tableName, s.columnName)
+		if _, err = sess.Exec(updateSQL, encrypted, nowInUTC(), row.Id); err != nil {
+			anyFailure = true
+			logger.Warn("Could not update secret while rolling it back", "table", s.tableName, "id", row.Id, "error", err)
+			continue
 		}
 	}
 
-	logger.Infof("Column %s from %s have been rolled back successfully\n", s.columnName, s.tableName)
+	if anyFailure {
+		logger.Warn(fmt.Sprintf("Column %s from %s has been rolled back with errors", s.columnName, s.tableName))
+	} else {
+		logger.Info(fmt.Sprintf("Column %s from %s has been rolled back successfully", s.columnName, s.tableName))
+	}
 
-	return nil
+	return anyFailure
+}
+
+func (s b64Secret) rollback(
+	secretsSrv *manager.SecretsService,
+	encryptionSrv encryption.Internal,
+	sess *xorm.Session,
+	secretKey string,
+) (anyFailure bool) {
+	var rows []struct {
+		Id     int
+		Secret string
+	}
+
+	if err := sess.Table(s.tableName).Select(fmt.Sprintf("id, %s as secret", s.columnName)).Find(&rows); err != nil {
+		logger.Warn("Could not find any secret to roll back", "table", s.tableName)
+		return true
+	}
+
+	for _, row := range rows {
+		if len(row.Secret) == 0 {
+			continue
+		}
+
+		decoded, err := base64.StdEncoding.DecodeString(row.Secret)
+		if err != nil {
+			anyFailure = true
+			logger.Warn("Could not decode base64-encoded secret while rolling it back", "table", s.tableName, "id", row.Id, "error", err)
+			continue
+		}
+
+		decrypted, err := secretsSrv.Decrypt(context.Background(), decoded)
+		if err != nil {
+			anyFailure = true
+			logger.Warn("Could not decrypt secret while rolling it back", "table", s.tableName, "id", row.Id, "error", err)
+			continue
+		}
+
+		encrypted, err := encryptionSrv.Encrypt(context.Background(), decrypted, secretKey)
+		if err != nil {
+			anyFailure = true
+			logger.Warn("Could not encrypt secret while rolling it back", "table", s.tableName, "id", row.Id, "error", err)
+			continue
+		}
+
+		encoded := base64.StdEncoding.EncodeToString(encrypted)
+		updateSQL := fmt.Sprintf("UPDATE %s SET %s = ? WHERE id = ?", s.tableName, s.columnName)
+		if _, err := sess.Exec(updateSQL, encoded, row.Id); err != nil {
+			anyFailure = true
+			logger.Warn("Could not update secret while rolling it back", "table", s.tableName, "id", row.Id, "error", err)
+			continue
+		}
+	}
+
+	if anyFailure {
+		logger.Warn(fmt.Sprintf("Column %s from %s has been rolled back with errors", s.columnName, s.tableName))
+	} else {
+		logger.Info(fmt.Sprintf("Column %s from %s has been rolled back successfully", s.columnName, s.tableName))
+	}
+
+	return anyFailure
 }
 
 func (s jsonSecret) rollback(
@@ -81,14 +134,15 @@ func (s jsonSecret) rollback(
 	encryptionSrv encryption.Internal,
 	sess *xorm.Session,
 	secretKey string,
-) error {
+) (anyFailure bool) {
 	var rows []struct {
 		Id             int
 		SecureJsonData map[string][]byte
 	}
 
 	if err := sess.Table(s.tableName).Cols("id", "secure_json_data").Find(&rows); err != nil {
-		return err
+		logger.Warn("Could not find any secret to roll back", "table", s.tableName)
+		return true
 	}
 
 	for _, row := range rows {
@@ -98,26 +152,35 @@ func (s jsonSecret) rollback(
 
 		decrypted, err := secretsSrv.DecryptJsonData(context.Background(), row.SecureJsonData)
 		if err != nil {
-			return err
+			anyFailure = true
+			logger.Warn("Could not decrypt secrets while rolling them back", "table", s.tableName, "id", row.Id, "error", err)
+			continue
 		}
 
-		var toUpdate struct {
+		toUpdate := struct {
 			SecureJsonData map[string][]byte
-		}
+			Updated        string
+		}{Updated: nowInUTC()}
 
 		toUpdate.SecureJsonData, err = encryptionSrv.EncryptJsonData(context.Background(), decrypted, secretKey)
 		if err != nil {
-			return err
+			logger.Warn("Could not re-encrypt secrets while rolling them back", "table", s.tableName, "id", row.Id, "error", err)
+			continue
 		}
 
 		if _, err := sess.Table(s.tableName).Where("id = ?", row.Id).Update(toUpdate); err != nil {
-			return err
+			logger.Warn("Could not update secrets while rolling them back", "table", s.tableName, "id", row.Id, "error", err)
+			continue
 		}
 	}
 
-	logger.Infof("Secure json data from %s have been rolled back successfully\n", s.tableName)
+	if anyFailure {
+		logger.Warn(fmt.Sprintf("Secure json data secrets from %s have been rolled back with errors", s.tableName))
+	} else {
+		logger.Info(fmt.Sprintf("Secure json data secrets from %s have been rolled back successfully", s.tableName))
+	}
 
-	return nil
+	return anyFailure
 }
 
 func (s alertingSecret) rollback(
@@ -125,7 +188,7 @@ func (s alertingSecret) rollback(
 	encryptionSrv encryption.Internal,
 	sess *xorm.Session,
 	secretKey string,
-) error {
+) (anyFailure bool) {
 	var results []struct {
 		Id                        int
 		AlertmanagerConfiguration string
@@ -133,14 +196,17 @@ func (s alertingSecret) rollback(
 
 	selectSQL := "SELECT id, alertmanager_configuration FROM alert_configuration"
 	if err := sess.SQL(selectSQL).Find(&results); err != nil {
-		return err
+		logger.Warn("Could not find any alert_configuration secret to roll back")
+		return true
 	}
 
 	for _, result := range results {
 		result := result
 		postableUserConfig, err := notifier.Load([]byte(result.AlertmanagerConfiguration))
 		if err != nil {
-			return err
+			anyFailure = true
+			logger.Warn("Could not load configuration (alert_configuration with id: %d) while rolling it back", result.Id, err)
+			continue
 		}
 
 		for _, receiver := range postableUserConfig.AlertmanagerConfig.Receivers {
@@ -148,17 +214,23 @@ func (s alertingSecret) rollback(
 				for k, v := range gmr.SecureSettings {
 					decoded, err := base64.StdEncoding.DecodeString(v)
 					if err != nil {
-						return err
+						anyFailure = true
+						logger.Warn("Could not decode base64-encoded secret (alert_configuration with id: %d, key)", k, result.Id, err)
+						continue
 					}
 
 					decrypted, err := secretsSrv.Decrypt(context.Background(), decoded)
 					if err != nil {
-						return err
+						anyFailure = true
+						logger.Warn("Could not decrypt secret (alert_configuration with id: %d, key)", k, result.Id, err)
+						continue
 					}
 
 					reencrypted, err := encryptionSrv.Encrypt(context.Background(), decrypted, secretKey)
 					if err != nil {
-						return err
+						anyFailure = true
+						logger.Warn("Could not re-encrypt secret (alert_configuration with id: %d, key)", k, result.Id, err)
+						continue
 					}
 
 					gmr.SecureSettings[k] = base64.StdEncoding.EncodeToString(reencrypted)
@@ -168,18 +240,26 @@ func (s alertingSecret) rollback(
 
 		marshalled, err := json.Marshal(postableUserConfig)
 		if err != nil {
-			return err
+			anyFailure = true
+			logger.Warn("Could not marshal configuration (alert_configuration with id: %d) while rolling it back", result.Id, err)
+			continue
 		}
 
 		result.AlertmanagerConfiguration = string(marshalled)
 		if _, err := sess.Table("alert_configuration").Where("id = ?", result.Id).Update(&result); err != nil {
-			return err
+			anyFailure = true
+			logger.Warn("Could not update secret (alert_configuration with id: %d) while rolling it back", result.Id, err)
+			continue
 		}
 	}
 
-	logger.Info("Alerting secrets have rolled re-encrypted successfully\n")
+	if anyFailure {
+		logger.Warn("Alerting configuration secrets have been rolled back with errors")
+	} else {
+		logger.Info("Alerting configuration secrets have been rolled back successfully")
+	}
 
-	return nil
+	return anyFailure
 }
 
 func RollBackSecrets(_ utils.CommandLine, runner runner.Runner) error {
@@ -188,31 +268,41 @@ func RollBackSecrets(_ utils.CommandLine, runner runner.Runner) error {
 		return nil
 	}
 
-	toMigrate := []interface {
-		rollback(*manager.SecretsService, encryption.Internal, *xorm.Session, string) error
+	toRollback := []interface {
+		rollback(*manager.SecretsService, encryption.Internal, *xorm.Session, string) bool
 	}{
-		simpleSecret{tableName: "dashboard_snapshot", columnName: "dashboard_encrypted", isBase64Encoded: false},
-		simpleSecret{tableName: "user_auth", columnName: "o_auth_access_token", isBase64Encoded: true},
-		simpleSecret{tableName: "user_auth", columnName: "o_auth_refresh_token", isBase64Encoded: true},
-		simpleSecret{tableName: "user_auth", columnName: "o_auth_token_type", isBase64Encoded: true},
+		simpleSecret{tableName: "dashboard_snapshot", columnName: "dashboard_encrypted"},
+		b64Secret{simpleSecret{tableName: "user_auth", columnName: "o_auth_access_token"}},
+		b64Secret{simpleSecret{tableName: "user_auth", columnName: "o_auth_refresh_token"}},
+		b64Secret{simpleSecret{tableName: "user_auth", columnName: "o_auth_token_type"}},
 		jsonSecret{tableName: "data_source"},
 		jsonSecret{tableName: "plugin_setting"},
 		alertingSecret{},
 	}
 
-	return runner.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
-		for _, m := range toMigrate {
-			if err := m.rollback(
-				runner.SecretsService,
-				runner.EncryptionService,
-				sess.Session,
-				runner.Cfg.SecretKey); err != nil {
-				return err
+	return runner.SQLStore.WithTransactionalDbSession(context.Background(), func(sess *sqlstore.DBSession) (err error) {
+		defer func() {
+			if r := recover(); r != nil {
+				err = errors.New(fmt.Sprint(r))
+				logger.Error("Secrets roll back failed, rolling back transaction...", "error", err)
+			}
+		}()
+
+		var anyFailure bool
+
+		for _, r := range toRollback {
+			if failed := r.rollback(runner.SecretsService, runner.EncryptionService, sess.Session, runner.Cfg.SecretKey); failed {
+				anyFailure = true
 			}
 		}
 
-		if _, err := sess.Exec("DELETE FROM data_keys"); err != nil {
-			logger.Warn("Error while cleaning up data keys table...", "err", err)
+		if anyFailure {
+			logger.Warn("Some errors happened, not cleaning up data keys table...")
+			return nil
+		}
+
+		if _, sqlErr := sess.Exec("DELETE FROM data_keys"); sqlErr != nil {
+			logger.Warn("Error while cleaning up data keys table...", "error", sqlErr)
 		}
 
 		return nil

pkg/cmd/grafana-cli/commands/secretsmigrations/secretsmigrations.go

@@ -0,0 +1,28 @@
+package secretsmigrations
+
+import (
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+)
+
+type simpleSecret struct {
+	tableName  string
+	columnName string
+}
+
+type b64Secret struct {
+	simpleSecret
+}
+
+type jsonSecret struct {
+	tableName string
+}
+
+type alertingSecret struct{}
+
+func nowInUTC() string {
+	return time.Now().UTC().Format("2006-01-02 15:04:05")
+}
+
+var logger = log.New("secrets.migrations")

pkg/cmd/grafana-cli/commands/secretsmigrations/types.go

@@ -1,13 +0,0 @@
-package secretsmigrations
-
-type simpleSecret struct {
-	tableName       string
-	columnName      string
-	isBase64Encoded bool
-}
-
-type jsonSecret struct {
-	tableName string
-}
-
-type alertingSecret struct{}

pkg/services/kmsproviders/kmsproviders.go

@@ -17,3 +17,11 @@ const (
 type Service interface {
 	Provide() (map[secrets.ProviderID]secrets.Provider, error)
 }
+
+func NormalizeProviderID(id secrets.ProviderID) secrets.ProviderID {
+	if id == Legacy {
+		return Default
+	}
+
+	return id
+}

pkg/services/secrets/database/database.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/kmsproviders"
 	"github.com/grafana/grafana/pkg/services/secrets"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"xorm.io/xorm"
@@ -100,26 +101,50 @@ func (ss *SecretsStoreImpl) ReEncryptDataKeys(
 		}
 
 		for _, k := range keys {
-			provider, ok := providers[k.Provider]
+			provider, ok := providers[kmsproviders.NormalizeProviderID(k.Provider)]
 			if !ok {
-				return fmt.Errorf("could not find encryption provider '%s'", k.Provider)
+				ss.log.Warn(
+					"Could not find provider to re-encrypt data encryption key",
+					"key_id", k.Name,
+					"provider", k.Provider,
+				)
+				continue
 			}
 
 			decrypted, err := provider.Decrypt(ctx, k.EncryptedData)
 			if err != nil {
-				return err
+				ss.log.Warn(
+					"Error while decrypting data encryption key to re-encrypt it",
+					"key_id", k.Name,
+					"provider", k.Provider,
+					"err", err,
+				)
+				continue
 			}
 
 			// Updating current data key by re-encrypting it with current provider.
 			// Accessing the current provider within providers map should be safe.
 			k.Provider = currProvider
+			k.Updated = time.Now()
 			k.EncryptedData, err = providers[currProvider].Encrypt(ctx, decrypted)
 			if err != nil {
-				return err
+				ss.log.Warn(
+					"Error while re-encrypting data encryption key",
+					"key_id", k.Name,
+					"provider", k.Provider,
+					"err", err,
+				)
+				continue
 			}
 
 			if _, err := sess.Table(dataKeysTable).Where("name = ?", k.Name).Update(k); err != nil {
-				return err
+				ss.log.Warn(
+					"Error while re-encrypting data encryption key",
+					"key_id", k.Name,
+					"provider", k.Provider,
+					"err", err,
+				)
+				continue
 			}
 		}
 

pkg/services/secrets/manager/manager.go

@@ -48,7 +48,7 @@ func ProvideSecretsService(
 
 	logger := log.New("secrets")
 	enabled := features.IsEnabled(featuremgmt.FlagEnvelopeEncryption)
-	currentProviderID := normalizeProviderID(secrets.ProviderID(
+	currentProviderID := kmsproviders.NormalizeProviderID(secrets.ProviderID(
 		settings.KeyValue("security", "encryption_provider").MustString(kmsproviders.Default),
 	))
 
@@ -79,14 +79,6 @@ func ProvideSecretsService(
 	return s, nil
 }
 
-func normalizeProviderID(id secrets.ProviderID) secrets.ProviderID {
-	if id == kmsproviders.Legacy {
-		return kmsproviders.Default
-	}
-
-	return id
-}
-
 func (s *SecretsService) registerUsageMetrics() {
 	s.usageStats.RegisterMetricsFunc(func(context.Context) (map[string]interface{}, error) {
 		usageMetrics := make(map[string]interface{})
@@ -330,7 +322,7 @@ func (s *SecretsService) dataKey(ctx context.Context, name string) ([]byte, erro
 	}
 
 	// 2. decrypt data key
-	provider, exists := s.providers[normalizeProviderID(dataKey.Provider)]
+	provider, exists := s.providers[kmsproviders.NormalizeProviderID(dataKey.Provider)]
 	if !exists {
 		return nil, fmt.Errorf("could not find encryption provider '%s'", dataKey.Provider)
 	}