Envelope encryption CLI migration command (#41454)

* Working version of migrate secrets CLI command

* Move user oauth info encryption away from db transaction

* Enable a mechanism for re-use db session on encryption

* De-duplicate shared code between db and runner commands

* Set up Wire build graph

* Remove enterprise Wire set

* Fix cylic dependency: sqlstore.DBSession <-> xorm.Session

* Minor fix (add missing base64 encoding)

* Extract CLI 'secrets-migration' commands from 'data-migration' ones

* Move runner package outside commands

* Update Makefile (gen-go path)

* Minor prettier fix

* Some minor XORM related refactors

* Include new Wire enterprise file into .gitignore

* Update Wire deps

Co-authored-by: Agnès Toulet <35176601+AgnesToulet@users.noreply.github.com>

Co-authored-by: Tania B <yalyna.ts@gmail.com>
Co-authored-by: Agnès Toulet <35176601+AgnesToulet@users.noreply.github.com>

pkg/cmd/grafana-cli/commands/commands.go

@@ -6,6 +6,7 @@ import (
 	"github.com/fatih/color"
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/commands/datamigrations"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/commands/secretsmigrations"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/runner"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
@@ -17,7 +18,6 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
-// nolint: unused,deadcode
 func runRunnerCommand(command func(commandLine utils.CommandLine, runner runner.Runner) error) func(context *cli.Context) error {
 	return func(context *cli.Context) error {
 		cmd := &utils.ContextCommandLine{Context: context}
@@ -160,7 +160,7 @@ var adminCommands = []*cli.Command{
 	},
 	{
 		Name:  "data-migration",
-		Usage: "Runs a script that migrates or cleanups data in your db",
+		Usage: "Runs a script that migrates or cleanups data in your database",
 		Subcommands: []*cli.Command{
 			{
 				Name:   "encrypt-datasource-passwords",
@@ -169,6 +169,17 @@ var adminCommands = []*cli.Command{
 			},
 		},
 	},
+	{
+		Name:  "secrets-migration",
+		Usage: "Runs a script that migrates secrets in your database",
+		Subcommands: []*cli.Command{
+			{
+				Name:   "re-encrypt",
+				Usage:  "Re-encrypts secrets by decrypting and re-encrypting them with the currently configured encryption. Returns ok unless there is an error. Safe to execute multiple times.",
+				Action: runRunnerCommand(secretsmigrations.ReEncryptSecrets),
+			},
+		},
+	},
 }
 
 var cueCommands = []*cli.Command{

pkg/cmd/grafana-cli/commands/secretsmigrations/reencrypt_secrets.go

@@ -0,0 +1,206 @@
+package secretsmigrations
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/runner"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
+	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
+	"github.com/grafana/grafana/pkg/services/secrets"
+	"github.com/grafana/grafana/pkg/services/secrets/manager"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"xorm.io/xorm"
+)
+
+type simpleSecret struct {
+	tableName       string
+	columnName      string
+	isBase64Encoded bool
+}
+
+func (s simpleSecret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm.Session) error {
+	var rows []struct {
+		Id     int
+		Secret string
+	}
+
+	if err := sess.Table(s.tableName).Select(fmt.Sprintf("id, %s as secret", s.columnName)).Find(&rows); err != nil {
+		return err
+	}
+
+	for _, row := range rows {
+		if len(row.Secret) == 0 {
+			continue
+		}
+
+		var (
+			err     error
+			decoded = []byte(row.Secret)
+		)
+
+		if s.isBase64Encoded {
+			decoded, err = base64.StdEncoding.DecodeString(row.Secret)
+			if err != nil {
+				return err
+			}
+		}
+
+		decrypted, err := secretsSrv.Decrypt(context.Background(), decoded)
+		if err != nil {
+			return err
+		}
+
+		encrypted, err := secretsSrv.EncryptWithDBSession(context.Background(), decrypted, secrets.WithoutScope(), sess)
+		if err != nil {
+			return err
+		}
+
+		encoded := string(encrypted)
+		if s.isBase64Encoded {
+			encoded = base64.StdEncoding.EncodeToString(encrypted)
+		}
+
+		updateSQL := fmt.Sprintf("UPDATE %s SET %s = ? WHERE id = ?", s.tableName, s.columnName)
+		if _, err := sess.Exec(updateSQL, encoded, row.Id); err != nil {
+			return err
+		}
+	}
+
+	logger.Infof("Column %s from %s has been re-encrypted successfully\n", s.columnName, s.tableName)
+
+	return nil
+}
+
+type jsonSecret struct {
+	tableName string
+}
+
+func (s jsonSecret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm.Session) error {
+	var rows []struct {
+		Id             int
+		SecureJsonData map[string][]byte
+	}
+
+	if err := sess.Table(s.tableName).Cols("id", "secure_json_data").Find(&rows); err != nil {
+		return err
+	}
+
+	for _, row := range rows {
+		if len(row.SecureJsonData) == 0 {
+			continue
+		}
+
+		decrypted, err := secretsSrv.DecryptJsonData(context.Background(), row.SecureJsonData)
+		if err != nil {
+			return err
+		}
+
+		var toUpdate struct {
+			SecureJsonData map[string][]byte
+		}
+
+		toUpdate.SecureJsonData, err = secretsSrv.EncryptJsonDataWithDBSession(context.Background(), decrypted, secrets.WithoutScope(), sess)
+		if err != nil {
+			return err
+		}
+
+		if _, err := sess.Table(s.tableName).Where("id = ?", row.Id).Update(toUpdate); err != nil {
+			return err
+		}
+	}
+
+	logger.Infof("Secure json data from %s has been re-encrypted successfully\n", s.tableName)
+
+	return nil
+}
+
+type alertingSecret struct{}
+
+func (s alertingSecret) reencrypt(secretsSrv *manager.SecretsService, sess *xorm.Session) error {
+	var results []struct {
+		Id                        int
+		AlertmanagerConfiguration []byte
+	}
+
+	selectSQL := "SELECT id, alertmanager_configuration FROM alert_configuration"
+	if err := sess.SQL(selectSQL).Find(&results); err != nil {
+		return err
+	}
+
+	for _, result := range results {
+		result := result
+		postableUserConfig, err := notifier.Load(result.AlertmanagerConfiguration)
+		if err != nil {
+			return err
+		}
+
+		for _, receiver := range postableUserConfig.AlertmanagerConfig.Receivers {
+			for _, gmr := range receiver.GrafanaManagedReceivers {
+				for k, v := range gmr.SecureSettings {
+					decoded, err := base64.StdEncoding.DecodeString(v)
+					if err != nil {
+						return err
+					}
+
+					decrypted, err := secretsSrv.Decrypt(context.Background(), decoded)
+					if err != nil {
+						return err
+					}
+
+					reencrypted, err := secretsSrv.EncryptWithDBSession(context.Background(), decrypted, secrets.WithoutScope(), sess)
+					if err != nil {
+						return err
+					}
+
+					gmr.SecureSettings[k] = base64.StdEncoding.EncodeToString(reencrypted)
+				}
+			}
+		}
+
+		result.AlertmanagerConfiguration, err = json.Marshal(postableUserConfig)
+		if err != nil {
+			return err
+		}
+
+		if _, err := sess.Table("alert_configuration").Where("id = ?", result.Id).Update(&result); err != nil {
+			return err
+		}
+	}
+
+	logger.Info("Alerting secrets has been re-encrypted successfully\n")
+
+	return nil
+}
+
+func ReEncryptSecrets(_ utils.CommandLine, runner runner.Runner) error {
+	if !runner.SettingsProvider.IsFeatureToggleEnabled(secrets.EnvelopeEncryptionFeatureToggle) {
+		logger.Warn("Envelope encryption is not enabled, quitting...")
+		return nil
+	}
+
+	toMigrate := []interface {
+		reencrypt(*manager.SecretsService, *xorm.Session) error
+	}{
+		simpleSecret{tableName: "dashboard_snapshot", columnName: "dashboard_encrypted", isBase64Encoded: false},
+		simpleSecret{tableName: "user_auth", columnName: "o_auth_access_token", isBase64Encoded: true},
+		simpleSecret{tableName: "user_auth", columnName: "o_auth_refresh_token", isBase64Encoded: true},
+		simpleSecret{tableName: "user_auth", columnName: "o_auth_token_type", isBase64Encoded: true},
+		jsonSecret{tableName: "data_source"},
+		jsonSecret{tableName: "plugin_setting"},
+		alertingSecret{},
+	}
+
+	return runner.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
+		for _, m := range toMigrate {
+			if err := m.reencrypt(runner.SecretsService, sess.Session); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}

pkg/cmd/grafana-cli/runner/runner.go

@@ -2,7 +2,7 @@ package runner
 
 import (
 	"github.com/grafana/grafana/pkg/services/encryption"
-	"github.com/grafana/grafana/pkg/services/secrets"
+	"github.com/grafana/grafana/pkg/services/secrets/manager"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/setting"
 )
@@ -12,11 +12,11 @@ type Runner struct {
 	SQLStore          *sqlstore.SQLStore
 	SettingsProvider  setting.Provider
 	EncryptionService encryption.Internal
-	SecretsService    secrets.Service
+	SecretsService    *manager.SecretsService
 }
 
 func New(cfg *setting.Cfg, sqlStore *sqlstore.SQLStore, settingsProvider setting.Provider,
-	encryptionService encryption.Internal, secretsService secrets.Service) Runner {
+	encryptionService encryption.Internal, secretsService *manager.SecretsService) Runner {
 	return Runner{
 		Cfg:               cfg,
 		SQLStore:          sqlStore,