Encryption: Move secrets migrations into secrets.Migrator (#51014)

pkg/api/http_server.go

@@ -167,6 +167,7 @@ type HTTPServer struct {
 	CoremodelRegistry            *registry.Generic
 	CoremodelStaticRegistry      *registry.Static
 	kvStore                      kvstore.KVStore
+	secretsMigrator              secrets.Migrator
 }
 
 type ServerOptions struct {
@@ -201,7 +202,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 	teamsPermissionsService accesscontrol.TeamPermissionsService, folderPermissionsService accesscontrol.FolderPermissionsService,
 	dashboardPermissionsService accesscontrol.DashboardPermissionsService, dashboardVersionService dashver.Service,
 	starService star.Service, csrfService csrf.Service, coremodelRegistry *registry.Generic, coremodelStaticRegistry *registry.Static,
-	kvStore kvstore.KVStore, remoteSecretsCheck secretsKV.UseRemoteSecretsPluginCheck,
+	kvStore kvstore.KVStore, secretsMigrator secrets.Migrator, remoteSecretsCheck secretsKV.UseRemoteSecretsPluginCheck,
 ) (*HTTPServer, error) {
 	web.Env = cfg.Env
 	m := web.New()
@@ -286,6 +287,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 		CoremodelRegistry:            coremodelRegistry,
 		CoremodelStaticRegistry:      coremodelStaticRegistry,
 		kvStore:                      kvStore,
+		secretsMigrator:              secretsMigrator,
 	}
 	if hs.Listener != nil {
 		hs.log.Debug("Using provided listener")

pkg/cmd/grafana-cli/commands/secretsmigrations/reencrypt_deks.go

@@ -1,18 +0,0 @@
-package secretsmigrations
-
-import (
-	"context"
-
-	"github.com/grafana/grafana/pkg/cmd/grafana-cli/runner"
-	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
-	"github.com/grafana/grafana/pkg/services/featuremgmt"
-)
-
-func ReEncryptDEKS(_ utils.CommandLine, runner runner.Runner) error {
-	if runner.Features.IsEnabled(featuremgmt.FlagDisableEnvelopeEncryption) {
-		logger.Warn("Envelope encryption is not enabled, quitting...")
-		return nil
-	}
-
-	return runner.SecretsService.ReEncryptDataKeys(context.Background())
-}

pkg/cmd/grafana-cli/commands/secretsmigrations/secretsmigrations.go

@@ -1,31 +1,39 @@
 package secretsmigrations
 
 import (
-	"encoding/base64"
-	"time"
+	"context"
 
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/runner"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
 )
 
-type simpleSecret struct {
-	tableName  string
-	columnName string
-}
-
-type b64Secret struct {
-	simpleSecret
-	hasUpdatedColumn bool
-	encoding         *base64.Encoding
-}
-
-type jsonSecret struct {
-	tableName string
-}
-
-type alertingSecret struct{}
-
-func nowInUTC() string {
-	return time.Now().UTC().Format("2006-01-02 15:04:05")
-}
-
 var logger = log.New("secrets.migrations")
+
+func ReEncryptDEKS(_ utils.CommandLine, runner runner.Runner) error {
+	if runner.Features.IsEnabled(featuremgmt.FlagDisableEnvelopeEncryption) {
+		logger.Warn("Envelope encryption is not enabled, quitting...")
+		return nil
+	}
+
+	return runner.SecretsService.ReEncryptDataKeys(context.Background())
+}
+
+func ReEncryptSecrets(_ utils.CommandLine, runner runner.Runner) error {
+	if runner.Features.IsEnabled(featuremgmt.FlagDisableEnvelopeEncryption) {
+		logger.Warn("Envelope encryption is not enabled, quitting...")
+		return nil
+	}
+
+	return runner.SecretsMigrator.ReEncryptSecrets(context.Background())
+}
+
+func RollBackSecrets(_ utils.CommandLine, runner runner.Runner) error {
+	if runner.Features.IsEnabled(featuremgmt.FlagDisableEnvelopeEncryption) {
+		logger.Warn("Envelope encryption is not enabled, quitting...")
+		return nil
+	}
+
+	return runner.SecretsMigrator.RollBackSecrets(context.Background())
+}

pkg/cmd/grafana-cli/runner/runner.go

@@ -3,6 +3,7 @@ package runner
 import (
 	"github.com/grafana/grafana/pkg/services/encryption"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/services/secrets"
 	"github.com/grafana/grafana/pkg/services/secrets/manager"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/setting"
@@ -15,16 +16,20 @@ type Runner struct {
 	Features          featuremgmt.FeatureToggles
 	EncryptionService encryption.Internal
 	SecretsService    *manager.SecretsService
+	SecretsMigrator   secrets.Migrator
 }
 
 func New(cfg *setting.Cfg, sqlStore *sqlstore.SQLStore, settingsProvider setting.Provider,
-	encryptionService encryption.Internal, features featuremgmt.FeatureToggles, secretsService *manager.SecretsService) Runner {
+	encryptionService encryption.Internal, features featuremgmt.FeatureToggles,
+	secretsService *manager.SecretsService, secretsMigrator secrets.Migrator,
+) Runner {
 	return Runner{
 		Cfg:               cfg,
 		SQLStore:          sqlStore,
 		SettingsProvider:  settingsProvider,
 		EncryptionService: encryptionService,
 		SecretsService:    secretsService,
+		SecretsMigrator:   secretsMigrator,
 		Features:          features,
 	}
 }

pkg/cmd/grafana-cli/runner/wire.go

@@ -17,6 +17,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/secrets"
 	secretsDatabase "github.com/grafana/grafana/pkg/services/secrets/database"
 	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
+	secretsMigrator "github.com/grafana/grafana/pkg/services/secrets/migrator"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/web"
@@ -37,6 +38,8 @@ var wireSet = wire.NewSet(
 	wire.Bind(new(secrets.Store), new(*secretsDatabase.SecretsStoreImpl)),
 	secretsManager.ProvideSecretsService,
 	wire.Bind(new(secrets.Service), new(*secretsManager.SecretsService)),
+	secretsMigrator.ProvideSecretsMigrator,
+	wire.Bind(new(secrets.Migrator), new(*secretsMigrator.SecretsMigrator)),
 	hooks.ProvideService,
 )
 

pkg/server/wire.go

@@ -87,6 +87,7 @@ import (
 	secretsDatabase "github.com/grafana/grafana/pkg/services/secrets/database"
 	secretsStore "github.com/grafana/grafana/pkg/services/secrets/kvstore"
 	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
+	secretsMigrator "github.com/grafana/grafana/pkg/services/secrets/migrator"
 	"github.com/grafana/grafana/pkg/services/serviceaccounts"
 	serviceaccountsmanager "github.com/grafana/grafana/pkg/services/serviceaccounts/manager"
 	"github.com/grafana/grafana/pkg/services/shorturls"
@@ -220,6 +221,8 @@ var wireBasicSet = wire.NewSet(
 	wire.Bind(new(secrets.Service), new(*secretsManager.SecretsService)),
 	secretsDatabase.ProvideSecretsStore,
 	wire.Bind(new(secrets.Store), new(*secretsDatabase.SecretsStoreImpl)),
+	secretsMigrator.ProvideSecretsMigrator,
+	wire.Bind(new(secrets.Migrator), new(*secretsMigrator.SecretsMigrator)),
 	grafanads.ProvideService,
 	wire.Bind(new(dashboardsnapshots.Store), new(*dashsnapstore.DashboardSnapshotStore)),
 	dashsnapstore.ProvideStore,

pkg/services/secrets/migrator/migrator.go

@@ -0,0 +1,117 @@
+package migrator
+
+import (
+	"context"
+	"encoding/base64"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/encryption"
+	"github.com/grafana/grafana/pkg/services/secrets/manager"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+type SecretsMigrator struct {
+	encryptionSrv encryption.Internal
+	secretsSrv    *manager.SecretsService
+	sqlStore      *sqlstore.SQLStore
+	settings      setting.Provider
+}
+
+func ProvideSecretsMigrator(
+	encryptionSrv encryption.Internal,
+	service *manager.SecretsService,
+	sqlStore *sqlstore.SQLStore,
+	settings setting.Provider,
+) *SecretsMigrator {
+	return &SecretsMigrator{
+		encryptionSrv: encryptionSrv,
+		secretsSrv:    service,
+		sqlStore:      sqlStore,
+		settings:      settings,
+	}
+}
+
+func (m *SecretsMigrator) ReEncryptSecrets(ctx context.Context) error {
+	toReencrypt := []interface {
+		reencrypt(context.Context, *manager.SecretsService, *sqlstore.SQLStore)
+	}{
+		simpleSecret{tableName: "dashboard_snapshot", columnName: "dashboard_encrypted"},
+		b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_access_token"}, encoding: base64.StdEncoding},
+		b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_refresh_token"}, encoding: base64.StdEncoding},
+		b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_token_type"}, encoding: base64.StdEncoding},
+		b64Secret{simpleSecret: simpleSecret{tableName: "secrets", columnName: "value"}, hasUpdatedColumn: true, encoding: base64.RawStdEncoding},
+		jsonSecret{tableName: "data_source"},
+		jsonSecret{tableName: "plugin_setting"},
+		alertingSecret{},
+	}
+
+	for _, r := range toReencrypt {
+		r.reencrypt(ctx, m.secretsSrv, m.sqlStore)
+	}
+
+	return nil
+}
+
+func (m *SecretsMigrator) RollBackSecrets(ctx context.Context) error {
+	toRollback := []interface {
+		rollback(context.Context, *manager.SecretsService, encryption.Internal, *sqlstore.SQLStore, string) bool
+	}{
+		simpleSecret{tableName: "dashboard_snapshot", columnName: "dashboard_encrypted"},
+		b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_access_token"}, encoding: base64.StdEncoding},
+		b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_refresh_token"}, encoding: base64.StdEncoding},
+		b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_token_type"}, encoding: base64.StdEncoding},
+		b64Secret{simpleSecret: simpleSecret{tableName: "secrets", columnName: "value"}, hasUpdatedColumn: true, encoding: base64.RawStdEncoding},
+		jsonSecret{tableName: "data_source"},
+		jsonSecret{tableName: "plugin_setting"},
+		alertingSecret{},
+	}
+
+	var anyFailure bool
+
+	for _, r := range toRollback {
+		if failed := r.rollback(ctx,
+			m.secretsSrv,
+			m.encryptionSrv,
+			m.sqlStore,
+			m.settings.KeyValue("security", "secret_key").Value(),
+		); failed {
+			anyFailure = true
+		}
+	}
+
+	if anyFailure {
+		logger.Warn("Some errors happened, not cleaning up data keys table...")
+		return nil
+	}
+
+	if _, sqlErr := m.sqlStore.NewSession(ctx).Exec("DELETE FROM data_keys"); sqlErr != nil {
+		logger.Warn("Error while cleaning up data keys table...", "error", sqlErr)
+	}
+
+	return nil
+}
+
+type simpleSecret struct {
+	tableName  string
+	columnName string
+}
+
+type b64Secret struct {
+	simpleSecret
+	hasUpdatedColumn bool
+	encoding         *base64.Encoding
+}
+
+type jsonSecret struct {
+	tableName string
+}
+
+type alertingSecret struct{}
+
+func nowInUTC() string {
+	return time.Now().UTC().Format("2006-01-02 15:04:05")
+}
+
+var logger = log.New("secrets.migrations")

pkg/services/secrets/migrator/reencrypt.go

@@ -1,4 +1,4 @@
-package secretsmigrations
+package migrator
 
 import (
 	"context"
@@ -6,9 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/grafana/grafana/pkg/cmd/grafana-cli/runner"
-	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
-	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
 	"github.com/grafana/grafana/pkg/services/secrets"
 	"github.com/grafana/grafana/pkg/services/secrets/manager"
@@ -265,29 +262,3 @@ func (s alertingSecret) reencrypt(ctx context.Context, secretsSrv *manager.Secre
 		logger.Info("Alerting configuration secrets have been re-encrypted successfully")
 	}
 }
-
-func ReEncryptSecrets(_ utils.CommandLine, runner runner.Runner) error {
-	if runner.Features.IsEnabled(featuremgmt.FlagDisableEnvelopeEncryption) {
-		logger.Warn("Envelope encryption is not enabled, quitting...")
-		return nil
-	}
-
-	toMigrate := []interface {
-		reencrypt(context.Context, *manager.SecretsService, *sqlstore.SQLStore)
-	}{
-		simpleSecret{tableName: "dashboard_snapshot", columnName: "dashboard_encrypted"},
-		b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_access_token"}, encoding: base64.StdEncoding},
-		b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_refresh_token"}, encoding: base64.StdEncoding},
-		b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_token_type"}, encoding: base64.StdEncoding},
-		b64Secret{simpleSecret: simpleSecret{tableName: "secrets", columnName: "value"}, hasUpdatedColumn: true, encoding: base64.RawStdEncoding},
-		jsonSecret{tableName: "data_source"},
-		jsonSecret{tableName: "plugin_setting"},
-		alertingSecret{},
-	}
-
-	for _, m := range toMigrate {
-		m.reencrypt(context.Background(), runner.SecretsService, runner.SQLStore)
-	}
-
-	return nil
-}

pkg/services/secrets/migrator/rollback.go

@@ -1,4 +1,4 @@
-package secretsmigrations
+package migrator
 
 import (
 	"context"
@@ -6,10 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/grafana/grafana/pkg/cmd/grafana-cli/runner"
-	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 	"github.com/grafana/grafana/pkg/services/encryption"
-	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
 	"github.com/grafana/grafana/pkg/services/secrets/manager"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
@@ -289,49 +286,3 @@ func (s alertingSecret) rollback(
 
 	return anyFailure
 }
-
-func RollBackSecrets(_ utils.CommandLine, runner runner.Runner) error {
-	if runner.Features.IsEnabled(featuremgmt.FlagDisableEnvelopeEncryption) {
-		logger.Warn("Envelope encryption is not enabled, quitting...")
-		return nil
-	}
-
-	toRollback := []interface {
-		rollback(context.Context, *manager.SecretsService, encryption.Internal, *sqlstore.SQLStore, string) bool
-	}{
-		simpleSecret{tableName: "dashboard_snapshot", columnName: "dashboard_encrypted"},
-		b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_access_token"}, encoding: base64.StdEncoding},
-		b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_refresh_token"}, encoding: base64.StdEncoding},
-		b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_token_type"}, encoding: base64.StdEncoding},
-		b64Secret{simpleSecret: simpleSecret{tableName: "secrets", columnName: "value"}, hasUpdatedColumn: true, encoding: base64.RawStdEncoding},
-		jsonSecret{tableName: "data_source"},
-		jsonSecret{tableName: "plugin_setting"},
-		alertingSecret{},
-	}
-
-	var anyFailure bool
-	ctx := context.Background()
-
-	for _, r := range toRollback {
-		if failed := r.rollback(
-			ctx,
-			runner.SecretsService,
-			runner.EncryptionService,
-			runner.SQLStore,
-			runner.Cfg.SecretKey,
-		); failed {
-			anyFailure = true
-		}
-	}
-
-	if anyFailure {
-		logger.Warn("Some errors happened, not cleaning up data keys table...")
-		return nil
-	}
-
-	if _, sqlErr := runner.SQLStore.NewSession(ctx).Exec("DELETE FROM data_keys"); sqlErr != nil {
-		logger.Warn("Error while cleaning up data keys table...", "error", sqlErr)
-	}
-
-	return nil
-}

pkg/services/secrets/secrets.go

@@ -69,3 +69,9 @@ func KeyLabel(scope string, providerID ProviderID) string {
 type BackgroundProvider interface {
 	Run(ctx context.Context) error
 }
+
+// Migrator is responsible for secrets migrations like re-encrypting or rolling back secrets.
+type Migrator interface {
+	ReEncryptSecrets(ctx context.Context) error
+	RollBackSecrets(ctx context.Context) error
+}