Chore: Refactor secrets kvstore to organize testing and migrations (#54249)

* Refactor migrations and tests for secrets kvstore

* Use fake secrets store as a shortcut on tests

* Update wire

* Use global migration logger

* Fix ds proxy tests

* Fix linting issues

* Rename data source test setup function

pkg/services/secrets/kvstore/kvstore.go

@@ -29,25 +29,18 @@ func ProvideService(
 ) (SecretsKVStore, error) {
 	var logger = log.New("secrets.kvstore")
 	var store SecretsKVStore
-	store = &secretsKVStoreSQL{
-		sqlStore:       sqlStore,
-		secretsService: secretsService,
-		log:            logger,
-		decryptionCache: decryptionCache{
-			cache: make(map[int64]cachedDecrypted),
-		},
-	}
+	store = NewSQLSecretsKVStore(sqlStore, secretsService, logger)
 	err := EvaluateRemoteSecretsPlugin(pluginsManager, cfg)
 	if err != nil {
 		logger.Debug(err.Error())
 	} else {
 		// Attempt to start the plugin
 		var secretsPlugin secretsmanagerplugin.SecretsManagerPlugin
-		secretsPlugin, err = startAndReturnPlugin(pluginsManager, context.Background())
+		secretsPlugin, err = StartAndReturnPlugin(pluginsManager, context.Background())
 		namespacedKVStore := GetNamespacedKVStore(kvstore)
 		if err != nil || secretsPlugin == nil {
 			logger.Error("failed to start remote secrets management plugin", "msg", err.Error())
-			if isFatal, readErr := isPluginStartupErrorFatal(context.Background(), namespacedKVStore); isFatal || readErr != nil {
+			if isFatal, readErr := IsPluginStartupErrorFatal(context.Background(), namespacedKVStore); isFatal || readErr != nil {
 				// plugin error was fatal or there was an error determining if the error was fatal
 				logger.Error("secrets management plugin is required to start -- exiting app")
 				if readErr != nil {
@@ -56,7 +49,7 @@ func ProvideService(
 				return nil, err
 			}
 		} else {
-			// as the plugin is installed, secretsKVStoreSQL is now replaced with
+			// as the plugin is installed, SecretsKVStoreSQL is now replaced with
 			// an instance of secretsKVStorePlugin with the sql store as a fallback
 			// (used for migration and in case a secret is not found).
 			store = &secretsKVStorePlugin{

pkg/services/secrets/kvstore/migrations/datasource_mig.go

@@ -0,0 +1,105 @@
+package migrations
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/grafana/grafana/pkg/infra/kvstore"
+	"github.com/grafana/grafana/pkg/services/datasources"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	secretskvs "github.com/grafana/grafana/pkg/services/secrets/kvstore"
+)
+
+const (
+	// Not set means migration has not happened
+	secretMigrationStatusKey = "secretMigrationStatus"
+	// Migration happened with disableSecretCompatibility set to false
+	compatibleSecretMigrationValue = "compatible"
+	// Migration happened with disableSecretCompatibility set to true
+	completeSecretMigrationValue = "complete"
+)
+
+type DataSourceSecretMigrationService struct {
+	dataSourcesService datasources.DataSourceService
+	kvStore            *kvstore.NamespacedKVStore
+	features           featuremgmt.FeatureToggles
+}
+
+func ProvideDataSourceMigrationService(
+	dataSourcesService datasources.DataSourceService,
+	kvStore kvstore.KVStore,
+	features featuremgmt.FeatureToggles,
+) *DataSourceSecretMigrationService {
+	return &DataSourceSecretMigrationService{
+		dataSourcesService: dataSourcesService,
+		kvStore:            kvstore.WithNamespace(kvStore, 0, secretskvs.DataSourceSecretType),
+		features:           features,
+	}
+}
+
+func (s *DataSourceSecretMigrationService) Migrate(ctx context.Context) error {
+	migrationStatus, _, err := s.kvStore.Get(ctx, secretMigrationStatusKey)
+	if err != nil {
+		return err
+	}
+	logger.Debug(fmt.Sprint("secret migration status is ", migrationStatus))
+	// If this flag is true, delete secrets from the legacy secrets store as they are migrated
+	disableSecretsCompatibility := s.features.IsEnabled(featuremgmt.FlagDisableSecretsCompatibility)
+	// If migration hasn't happened, migrate to unified secrets and keep copy in legacy
+	// If a complete migration happened and now backwards compatibility is enabled, copy secrets back to legacy
+	needCompatibility := migrationStatus != compatibleSecretMigrationValue && !disableSecretsCompatibility
+	// If migration hasn't happened, migrate to unified secrets and delete from legacy
+	// If a compatible migration happened and now compatibility is disabled, delete secrets from legacy
+	needMigration := migrationStatus != completeSecretMigrationValue && disableSecretsCompatibility
+
+	if needCompatibility || needMigration {
+		logger.Debug("performing secret migration", "needs migration", needMigration, "needs compatibility", needCompatibility)
+		query := &datasources.GetAllDataSourcesQuery{}
+		err := s.dataSourcesService.GetAllDataSources(ctx, query)
+		if err != nil {
+			return err
+		}
+
+		for _, ds := range query.Result {
+			secureJsonData, err := s.dataSourcesService.DecryptedValues(ctx, ds)
+			if err != nil {
+				return err
+			}
+
+			// Secrets are set by the update data source function if the SecureJsonData is set in the command
+			// Secrets are deleted by the update data source function if the disableSecretsCompatibility flag is enabled
+			err = s.dataSourcesService.UpdateDataSource(ctx, &datasources.UpdateDataSourceCommand{
+				Id:             ds.Id,
+				OrgId:          ds.OrgId,
+				Uid:            ds.Uid,
+				Name:           ds.Name,
+				JsonData:       ds.JsonData,
+				SecureJsonData: secureJsonData,
+
+				// These are needed by the SQL function due to UseBool and MustCols
+				IsDefault:       ds.IsDefault,
+				BasicAuth:       ds.BasicAuth,
+				WithCredentials: ds.WithCredentials,
+				ReadOnly:        ds.ReadOnly,
+				User:            ds.User,
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		var newMigStatus string
+		if disableSecretsCompatibility {
+			newMigStatus = completeSecretMigrationValue
+		} else {
+			newMigStatus = compatibleSecretMigrationValue
+		}
+		err = s.kvStore.Set(ctx, secretMigrationStatusKey, newMigStatus)
+		if err != nil {
+			return err
+		}
+		logger.Debug(fmt.Sprint("set secret migration status to ", newMigStatus))
+	}
+
+	return nil
+}

pkg/services/secrets/kvstore/migrations/datasource_mig_test.go

@@ -0,0 +1,346 @@
+package migrations
+
+import (
+	"context"
+	"testing"
+
+	"github.com/grafana/grafana/pkg/infra/kvstore"
+	"github.com/grafana/grafana/pkg/infra/log"
+	acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
+	"github.com/grafana/grafana/pkg/services/datasources"
+	dsservice "github.com/grafana/grafana/pkg/services/datasources/service"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/services/secrets/fakes"
+	secretskvs "github.com/grafana/grafana/pkg/services/secrets/kvstore"
+	secretsmng "github.com/grafana/grafana/pkg/services/secrets/manager"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/stretchr/testify/assert"
+)
+
+func SetupTestDataSourceSecretMigrationService(t *testing.T, sqlStore *sqlstore.SQLStore, kvStore kvstore.KVStore, secretsStore secretskvs.SecretsKVStore, compatibility bool) *DataSourceSecretMigrationService {
+	t.Helper()
+	cfg := &setting.Cfg{}
+	features := featuremgmt.WithFeatures()
+	if !compatibility {
+		features = featuremgmt.WithFeatures(featuremgmt.FlagDisableSecretsCompatibility, true)
+	}
+	secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
+	dsService := dsservice.ProvideService(sqlStore, secretsService, secretsStore, cfg, features, acmock.New().WithDisabled(), acmock.NewMockedPermissionsService())
+	migService := ProvideDataSourceMigrationService(dsService, kvStore, features)
+	return migService
+}
+
+func TestMigrate(t *testing.T) {
+	t.Run("should migrate from legacy to unified without compatibility", func(t *testing.T) {
+		sqlStore := sqlstore.InitTestDB(t)
+		kvStore := kvstore.ProvideService(sqlStore)
+		secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
+		secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
+		migService := SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, false)
+
+		dataSourceName := "Test"
+		dataSourceOrg := int64(1)
+
+		// Add test data source
+		err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
+			OrgId:  dataSourceOrg,
+			Name:   dataSourceName,
+			Type:   datasources.DS_MYSQL,
+			Access: datasources.DS_ACCESS_DIRECT,
+			Url:    "http://test",
+			EncryptedSecureJsonData: map[string][]byte{
+				"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
+			},
+		})
+		assert.NoError(t, err)
+
+		// Check if the secret json data was added
+		query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.NotEmpty(t, query.Result.SecureJsonData)
+
+		// Check if the migration status key is empty
+		value, exist, err := kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Check that the secret is not present on the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Run the migration
+		err = migService.Migrate(context.Background())
+		assert.NoError(t, err)
+
+		// Check if the secure json data was deleted
+		query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.Empty(t, query.Result.SecureJsonData)
+
+		// Check if the secret was added to the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, value)
+		assert.True(t, exist)
+
+		// Check if the migration status key was set
+		value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Equal(t, completeSecretMigrationValue, value)
+		assert.True(t, exist)
+	})
+
+	t.Run("should migrate from legacy to unified with compatibility", func(t *testing.T) {
+		sqlStore := sqlstore.InitTestDB(t)
+		kvStore := kvstore.ProvideService(sqlStore)
+		secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
+		secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
+		migService := SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, true)
+
+		dataSourceName := "Test"
+		dataSourceOrg := int64(1)
+
+		// Add test data source
+		err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
+			OrgId:  dataSourceOrg,
+			Name:   dataSourceName,
+			Type:   datasources.DS_MYSQL,
+			Access: datasources.DS_ACCESS_DIRECT,
+			Url:    "http://test",
+			EncryptedSecureJsonData: map[string][]byte{
+				"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
+			},
+		})
+		assert.NoError(t, err)
+
+		// Check if the secret json data was added
+		query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.NotEmpty(t, query.Result.SecureJsonData)
+
+		// Check if the migration status key is empty
+		value, exist, err := kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Check that the secret is not present on the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Run the migration
+		err = migService.Migrate(context.Background())
+		assert.NoError(t, err)
+
+		// Check if the secure json data was maintained for compatibility
+		query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.NotEmpty(t, query.Result.SecureJsonData)
+
+		// Check if the secret was added to the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, value)
+		assert.True(t, exist)
+
+		// Check if the migration status key was set
+		value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Equal(t, compatibleSecretMigrationValue, value)
+		assert.True(t, exist)
+	})
+
+	t.Run("should replicate from unified to legacy for compatibility", func(t *testing.T) {
+		sqlStore := sqlstore.InitTestDB(t)
+		kvStore := kvstore.ProvideService(sqlStore)
+		secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
+		secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
+		migService := SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, false)
+
+		dataSourceName := "Test"
+		dataSourceOrg := int64(1)
+
+		// Add test data source
+		err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
+			OrgId:  dataSourceOrg,
+			Name:   dataSourceName,
+			Type:   datasources.DS_MYSQL,
+			Access: datasources.DS_ACCESS_DIRECT,
+			Url:    "http://test",
+			EncryptedSecureJsonData: map[string][]byte{
+				"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
+			},
+		})
+		assert.NoError(t, err)
+
+		// Check if the secret json data was added
+		query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.NotEmpty(t, query.Result.SecureJsonData)
+
+		// Check if the migration status key is empty
+		value, exist, err := kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Check that the secret is not present on the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Run the migration without compatibility
+		err = migService.Migrate(context.Background())
+		assert.NoError(t, err)
+
+		// Check if the secure json data was deleted
+		query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.Empty(t, query.Result.SecureJsonData)
+
+		// Check if the secret was added to the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, value)
+		assert.True(t, exist)
+
+		// Check if the migration status key was set
+		value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Equal(t, completeSecretMigrationValue, value)
+		assert.True(t, exist)
+
+		// Run the migration with compatibility
+		migService = SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, true)
+		err = migService.Migrate(context.Background())
+		assert.NoError(t, err)
+
+		// Check if the secure json data was re-added for compatibility
+		query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.NotEmpty(t, query.Result.SecureJsonData)
+
+		// Check if the secret was added to the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, value)
+		assert.True(t, exist)
+
+		// Check if the migration status key was set
+		value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Equal(t, compatibleSecretMigrationValue, value)
+		assert.True(t, exist)
+	})
+
+	t.Run("should delete from legacy to remove compatibility", func(t *testing.T) {
+		sqlStore := sqlstore.InitTestDB(t)
+		kvStore := kvstore.ProvideService(sqlStore)
+		secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
+		secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
+		migService := SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, true)
+
+		dataSourceName := "Test"
+		dataSourceOrg := int64(1)
+
+		// Add test data source
+		err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
+			OrgId:  dataSourceOrg,
+			Name:   dataSourceName,
+			Type:   datasources.DS_MYSQL,
+			Access: datasources.DS_ACCESS_DIRECT,
+			Url:    "http://test",
+			EncryptedSecureJsonData: map[string][]byte{
+				"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
+			},
+		})
+		assert.NoError(t, err)
+
+		// Check if the secret json data was added
+		query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.NotEmpty(t, query.Result.SecureJsonData)
+
+		// Check if the migration status key is empty
+		value, exist, err := kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Check that the secret is not present on the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Run the migration with compatibility
+		err = migService.Migrate(context.Background())
+		assert.NoError(t, err)
+
+		// Check if the secure json data was maintained for compatibility
+		query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.NotEmpty(t, query.Result.SecureJsonData)
+
+		// Check if the secret was added to the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, value)
+		assert.True(t, exist)
+
+		// Check if the migration status key was set
+		value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Equal(t, compatibleSecretMigrationValue, value)
+		assert.True(t, exist)
+
+		// Run the migration without compatibility
+		migService = SetupTestDataSourceSecretMigrationService(t, sqlStore, kvStore, secretsStore, false)
+		err = migService.Migrate(context.Background())
+		assert.NoError(t, err)
+
+		// Check if the secure json data was deleted
+		query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.Empty(t, query.Result.SecureJsonData)
+
+		// Check if the secret was added to the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretskvs.DataSourceSecretType)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, value)
+		assert.True(t, exist)
+
+		// Check if the migration status key was set
+		value, exist, err = kvStore.Get(context.Background(), 0, secretskvs.DataSourceSecretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Equal(t, completeSecretMigrationValue, value)
+		assert.True(t, exist)
+	})
+}

pkg/services/secrets/kvstore/migrations/from_plugin_mig.go

@@ -1,15 +1,14 @@
-package kvstore
+package migrations
 
 import (
 	"context"
 	"fmt"
-	"sync"
 
 	"github.com/grafana/grafana/pkg/infra/kvstore"
-	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/plugins/backendplugin/secretsmanagerplugin"
 	"github.com/grafana/grafana/pkg/services/secrets"
+	secretskvs "github.com/grafana/grafana/pkg/services/secrets/kvstore"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/setting"
 )
@@ -17,7 +16,6 @@ import (
 // MigrateFromPluginService This migrator will handle migration of the configured plugin secrets back to Grafana unified secrets
 type MigrateFromPluginService struct {
 	cfg            *setting.Cfg
-	logger         log.Logger
 	sqlStore       sqlstore.Store
 	secretsService secrets.Service
 	manager        plugins.SecretsPluginManager
@@ -34,7 +32,6 @@ func ProvideMigrateFromPluginService(
 ) *MigrateFromPluginService {
 	return &MigrateFromPluginService{
 		cfg:            cfg,
-		logger:         log.New("sec-plugin-mig"),
 		sqlStore:       sqlStore,
 		secretsService: secretsService,
 		manager:        manager,
@@ -43,43 +40,36 @@ func ProvideMigrateFromPluginService(
 }
 
 func (s *MigrateFromPluginService) Migrate(ctx context.Context) error {
-	s.logger.Debug("starting migration of plugin secrets to unified secrets")
+	logger.Debug("starting migration of plugin secrets to unified secrets")
 	// access the plugin directly
-	plugin, err := startAndReturnPlugin(s.manager, context.Background())
+	plugin, err := secretskvs.StartAndReturnPlugin(s.manager, context.Background())
 	if err != nil {
-		s.logger.Error("Error retrieiving plugin", "error", err.Error())
+		logger.Error("Error retrieiving plugin", "error", err.Error())
 		return err
 	}
 	// Get full list of secrets from the plugin
 	res, err := plugin.GetAllSecrets(ctx, &secretsmanagerplugin.GetAllSecretsRequest{})
 	if err != nil {
-		s.logger.Error("Failed to retrieve all secrets from plugin")
+		logger.Error("Failed to retrieve all secrets from plugin")
 		return err
 	}
 	totalSecrets := len(res.Items)
-	s.logger.Debug("retrieved all secrets from plugin", "num secrets", totalSecrets)
+	logger.Debug("retrieved all secrets from plugin", "num secrets", totalSecrets)
 	// create a secret sql store manually
-	secretsSql := &secretsKVStoreSQL{
-		sqlStore:       s.sqlStore,
-		secretsService: s.secretsService,
-		log:            s.logger,
-		decryptionCache: decryptionCache{
-			cache: make(map[int64]cachedDecrypted),
-		},
-	}
+	secretsSql := secretskvs.NewSQLSecretsKVStore(s.sqlStore, s.secretsService, logger)
 	for i, item := range res.Items {
-		s.logger.Debug(fmt.Sprintf("Migrating secret %d of %d", i+1, totalSecrets), "current", i+1, "secretCount", totalSecrets)
+		logger.Debug(fmt.Sprintf("Migrating secret %d of %d", i+1, totalSecrets), "current", i+1, "secretCount", totalSecrets)
 		// Add to sql store
 		err = secretsSql.Set(ctx, item.Key.OrgId, item.Key.Namespace, item.Key.Type, item.Value)
 		if err != nil {
-			s.logger.Error("Error adding secret to unified secrets", "orgId", item.Key.OrgId,
+			logger.Error("Error adding secret to unified secrets", "orgId", item.Key.OrgId,
 				"namespace", item.Key.Namespace, "type", item.Key.Type)
 			return err
 		}
 	}
 
 	for i, item := range res.Items {
-		s.logger.Debug(fmt.Sprintf("Cleaning secret %d of %d", i+1, totalSecrets), "current", i+1, "secretCount", totalSecrets)
+		logger.Debug(fmt.Sprintf("Cleaning secret %d of %d", i+1, totalSecrets), "current", i+1, "secretCount", totalSecrets)
 		// Delete from the plugin
 		_, err := plugin.DeleteSecret(ctx, &secretsmanagerplugin.DeleteSecretRequest{
 			KeyDescriptor: &secretsmanagerplugin.Key{
@@ -88,28 +78,28 @@ func (s *MigrateFromPluginService) Migrate(ctx context.Context) error {
 				Type:      item.Key.Type,
 			}})
 		if err != nil {
-			s.logger.Error("Error deleting secret from plugin after migration", "orgId", item.Key.OrgId,
+			logger.Error("Error deleting secret from plugin after migration", "orgId", item.Key.OrgId,
 				"namespace", item.Key.Namespace, "type", item.Key.Type)
 			continue
 		}
 	}
-	s.logger.Debug("Completed migration of secrets from plugin")
+	logger.Debug("Completed migration of secrets from plugin")
 
 	// The plugin is no longer needed at the moment
-	err = setPluginStartupErrorFatal(ctx, GetNamespacedKVStore(s.kvstore), false)
+	err = secretskvs.SetPluginStartupErrorFatal(ctx, secretskvs.GetNamespacedKVStore(s.kvstore), false)
 	if err != nil {
-		s.logger.Error("Failed to remove plugin error fatal flag", "error", err.Error())
+		logger.Error("Failed to remove plugin error fatal flag", "error", err.Error())
 	}
 	// Reset the fatal flag setter in case another secret is created on the plugin
-	fatalFlagOnce = sync.Once{}
+	secretskvs.ResetPlugin()
 
-	s.logger.Debug("Shutting down secrets plugin now that migration is complete")
+	logger.Debug("Shutting down secrets plugin now that migration is complete")
 	// if `use_plugin` wasn't set, stop the plugin after migration
 	if !s.cfg.SectionWithEnvOverrides("secrets").Key("use_plugin").MustBool(false) {
 		err := s.manager.SecretsManager().Stop(ctx)
 		if err != nil {
 			// Log a warning but don't throw an error
-			s.logger.Error("Error stopping secrets plugin after migration", "error", err.Error())
+			logger.Error("Error stopping secrets plugin after migration", "error", err.Error())
 		}
 	}
 	return nil

pkg/services/secrets/kvstore/migrations/from_plugin_mig_test.go

@@ -1,4 +1,4 @@
-package kvstore
+package migrations
 
 import (
 	"context"
@@ -8,6 +8,7 @@ import (
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/plugins/backendplugin/secretsmanagerplugin"
 	"github.com/grafana/grafana/pkg/services/secrets/fakes"
+	secretskvs "github.com/grafana/grafana/pkg/services/secrets/kvstore"
 	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/setting"
@@ -39,13 +40,13 @@ func TestPluginSecretMigrationService_MigrateFromPlugin(t *testing.T) {
 }
 
 // Set up services used in migration
-func setupTestMigrateFromPluginService(t *testing.T) (*MigrateFromPluginService, secretsmanagerplugin.SecretsManagerPlugin, *secretsKVStoreSQL) {
+func setupTestMigrateFromPluginService(t *testing.T) (*MigrateFromPluginService, secretsmanagerplugin.SecretsManagerPlugin, *secretskvs.SecretsKVStoreSQL) {
 	t.Helper()
 
 	// this is to init the sql secret store inside the migration
 	sqlStore := sqlstore.InitTestDB(t)
 	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-	manager := NewFakeSecretsPluginManager(t, false)
+	manager := secretskvs.NewFakeSecretsPluginManager(t, false)
 	migratorService := ProvideMigrateFromPluginService(
 		setting.NewCfg(),
 		sqlStore,
@@ -54,14 +55,7 @@ func setupTestMigrateFromPluginService(t *testing.T) (*MigrateFromPluginService,
 		kvstore.ProvideService(sqlStore),
 	)
 
-	secretsSql := &secretsKVStoreSQL{
-		sqlStore:       sqlStore,
-		secretsService: secretsService,
-		log:            log.New("test.logger"),
-		decryptionCache: decryptionCache{
-			cache: make(map[int64]cachedDecrypted),
-		},
-	}
+	secretsSql := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
 
 	return migratorService, manager.SecretsManager().SecretsManager, secretsSql
 }
@@ -88,7 +82,7 @@ func validatePluginSecretsWereDeleted(t *testing.T, plugin secretsmanagerplugin.
 }
 
 // validates that secrets are in sql
-func validateSecretWasStoredInSql(t *testing.T, sqlStore *secretsKVStoreSQL, ctx context.Context, orgId int64, namespace string, typ string, expectedValue string) {
+func validateSecretWasStoredInSql(t *testing.T, sqlStore *secretskvs.SecretsKVStoreSQL, ctx context.Context, orgId int64, namespace string, typ string, expectedValue string) {
 	t.Helper()
 	res, exists, err := sqlStore.Get(ctx, orgId, namespace, typ)
 	require.NoError(t, err)

pkg/services/secrets/kvstore/migrations/migrator.go

@@ -7,8 +7,6 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/serverlock"
-	datasources "github.com/grafana/grafana/pkg/services/datasources/service"
-	"github.com/grafana/grafana/pkg/services/secrets/kvstore"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
@@ -24,16 +22,16 @@ type SecretMigrationService interface {
 type SecretMigrationServiceImpl struct {
 	services                 []SecretMigrationService
 	ServerLockService        *serverlock.ServerLockService
-	migrateToPluginService   *kvstore.MigrateToPluginService
-	migrateFromPluginService *kvstore.MigrateFromPluginService
+	migrateToPluginService   *MigrateToPluginService
+	migrateFromPluginService *MigrateFromPluginService
 }
 
 func ProvideSecretMigrationService(
 	cfg *setting.Cfg,
 	serverLockService *serverlock.ServerLockService,
-	dataSourceSecretMigrationService *datasources.DataSourceSecretMigrationService,
-	migrateToPluginService *kvstore.MigrateToPluginService,
-	migrateFromPluginService *kvstore.MigrateFromPluginService,
+	dataSourceSecretMigrationService *DataSourceSecretMigrationService,
+	migrateToPluginService *MigrateToPluginService,
+	migrateFromPluginService *MigrateFromPluginService,
 ) *SecretMigrationServiceImpl {
 	services := make([]SecretMigrationService, 0)
 	services = append(services, dataSourceSecretMigrationService)

pkg/services/secrets/kvstore/migrations/to_plugin_mig.go

@@ -1,4 +1,4 @@
-package kvstore
+package migrations
 
 import (
 	"context"
@@ -6,9 +6,9 @@ import (
 	"fmt"
 
 	"github.com/grafana/grafana/pkg/infra/kvstore"
-	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/services/secrets"
+	secretskvs "github.com/grafana/grafana/pkg/services/secrets/kvstore"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/setting"
 )
@@ -16,9 +16,8 @@ import (
 // MigrateToPluginService This migrator will handle migration of datasource secrets (aka Unified secrets)
 // into the plugin secrets configured
 type MigrateToPluginService struct {
-	secretsStore   SecretsKVStore
+	secretsStore   secretskvs.SecretsKVStore
 	cfg            *setting.Cfg
-	logger         log.Logger
 	sqlStore       sqlstore.Store
 	secretsService secrets.Service
 	kvstore        kvstore.KVStore
@@ -26,7 +25,7 @@ type MigrateToPluginService struct {
 }
 
 func ProvideMigrateToPluginService(
-	secretsStore SecretsKVStore,
+	secretsStore secretskvs.SecretsKVStore,
 	cfg *setting.Cfg,
 	sqlStore sqlstore.Store,
 	secretsService secrets.Service,
@@ -36,7 +35,6 @@ func ProvideMigrateToPluginService(
 	return &MigrateToPluginService{
 		secretsStore:   secretsStore,
 		cfg:            cfg,
-		logger:         log.New("secret.migration.plugin"),
 		sqlStore:       sqlStore,
 		secretsService: secretsService,
 		kvstore:        kvstore,
@@ -45,8 +43,8 @@ func ProvideMigrateToPluginService(
 }
 
 func (s *MigrateToPluginService) Migrate(ctx context.Context) error {
-	if err := EvaluateRemoteSecretsPlugin(s.manager, s.cfg); err == nil {
-		s.logger.Debug("starting migration of unified secrets to the plugin")
+	if err := secretskvs.EvaluateRemoteSecretsPlugin(s.manager, s.cfg); err == nil {
+		logger.Debug("starting migration of unified secrets to the plugin")
 		// we need to get the fallback store since in this scenario the secrets store would be the plugin.
 		fallbackStore := s.secretsStore.Fallback()
 		if fallbackStore == nil {
@@ -54,10 +52,10 @@ func (s *MigrateToPluginService) Migrate(ctx context.Context) error {
 		}
 
 		// before we start migrating, check see if plugin startup failures were already fatal
-		namespacedKVStore := GetNamespacedKVStore(s.kvstore)
-		wasFatal, err := isPluginStartupErrorFatal(ctx, namespacedKVStore)
+		namespacedKVStore := secretskvs.GetNamespacedKVStore(s.kvstore)
+		wasFatal, err := secretskvs.IsPluginStartupErrorFatal(ctx, namespacedKVStore)
 		if err != nil {
-			s.logger.Warn("unable to determine whether plugin startup failures are fatal - continuing migration anyway.")
+			logger.Warn("unable to determine whether plugin startup failures are fatal - continuing migration anyway.")
 		}
 
 		allSec, err := fallbackStore.GetAll(ctx)
@@ -66,35 +64,35 @@ func (s *MigrateToPluginService) Migrate(ctx context.Context) error {
 		}
 		totalSec := len(allSec)
 		// We just set it again as the current secret store should be the plugin secret
-		s.logger.Debug(fmt.Sprintf("Total amount of secrets to migrate: %d", totalSec))
+		logger.Debug(fmt.Sprintf("Total amount of secrets to migrate: %d", totalSec))
 		for i, sec := range allSec {
-			s.logger.Debug(fmt.Sprintf("Migrating secret %d of %d", i+1, totalSec), "current", i+1, "secretCount", totalSec)
+			logger.Debug(fmt.Sprintf("Migrating secret %d of %d", i+1, totalSec), "current", i+1, "secretCount", totalSec)
 			err = s.secretsStore.Set(ctx, *sec.OrgId, *sec.Namespace, *sec.Type, sec.Value)
 			if err != nil {
 				return err
 			}
 		}
-		s.logger.Debug("migrated unified secrets to plugin", "number of secrets", totalSec)
+		logger.Debug("migrated unified secrets to plugin", "number of secrets", totalSec)
 		// as no err was returned, when we delete all the secrets from the sql store
 		for index, sec := range allSec {
-			s.logger.Debug(fmt.Sprintf("Cleaning secret %d of %d", index+1, totalSec), "current", index+1, "secretCount", totalSec)
+			logger.Debug(fmt.Sprintf("Cleaning secret %d of %d", index+1, totalSec), "current", index+1, "secretCount", totalSec)
 
 			err = fallbackStore.Del(ctx, *sec.OrgId, *sec.Namespace, *sec.Type)
 			if err != nil {
-				s.logger.Error("plugin migrator encountered error while deleting unified secrets")
+				logger.Error("plugin migrator encountered error while deleting unified secrets")
 				if index == 0 && !wasFatal {
 					// old unified secrets still exists, so plugin startup errors are still not fatal, unless they were before we started
-					err := setPluginStartupErrorFatal(ctx, namespacedKVStore, false)
+					err := secretskvs.SetPluginStartupErrorFatal(ctx, namespacedKVStore, false)
 					if err != nil {
-						s.logger.Error("error reverting plugin failure fatal status", "error", err.Error())
+						logger.Error("error reverting plugin failure fatal status", "error", err.Error())
 					} else {
-						s.logger.Debug("application will continue to function without the secrets plugin")
+						logger.Debug("application will continue to function without the secrets plugin")
 					}
 				}
 				return err
 			}
 		}
-		s.logger.Debug("deleted unified secrets after migration", "number of secrets", totalSec)
+		logger.Debug("deleted unified secrets after migration", "number of secrets", totalSec)
 	}
 	return nil
 }

pkg/services/secrets/kvstore/migrations/to_plugin_mig_test.go

@@ -1,15 +1,19 @@
-package kvstore
+package migrations
 
 import (
 	"context"
+	"errors"
 	"testing"
 
 	"github.com/grafana/grafana/pkg/infra/kvstore"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/secrets/fakes"
+	secretskvs "github.com/grafana/grafana/pkg/services/secrets/kvstore"
 	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/services/sqlstore/mockstore"
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/ini.v1"
 )
@@ -43,14 +47,32 @@ func TestPluginSecretMigrationService_MigrateToPlugin(t *testing.T) {
 	})
 }
 
-func addSecretToSqlStore(t *testing.T, sqlSecretStore *secretsKVStoreSQL, ctx context.Context, orgId int64, namespace1 string, typ string, value string) {
+// With fatal flag unset, do a migration with backwards compatibility disabled. When unified secrets are deleted, return an error on the first deletion
+// Should result in the fatal flag remaining unset
+func TestFatalPluginErr_MigrationTestWithErrorDeletingUnifiedSecrets(t *testing.T) {
+	p, err := secretskvs.SetupFatalCrashTest(t, false, false, true)
+	assert.NoError(t, err)
+
+	migration := setupTestMigratorServiceWithDeletionError(t, p.SecretsKVStore, &mockstore.SQLStoreMock{
+		ExpectedError: errors.New("random error"),
+	}, p.KVStore)
+	err = migration.Migrate(context.Background())
+	assert.Error(t, err)
+	assert.Equal(t, "mocked del error", err.Error())
+
+	isFatal, err := secretskvs.IsPluginStartupErrorFatal(context.Background(), secretskvs.GetNamespacedKVStore(p.KVStore))
+	assert.NoError(t, err)
+	assert.False(t, isFatal)
+}
+
+func addSecretToSqlStore(t *testing.T, sqlSecretStore *secretskvs.SecretsKVStoreSQL, ctx context.Context, orgId int64, namespace1 string, typ string, value string) {
 	t.Helper()
 	err := sqlSecretStore.Set(ctx, orgId, namespace1, typ, value)
 	require.NoError(t, err)
 }
 
 // validates that secrets on the sql store were deleted.
-func validateSqlSecretWasDeleted(t *testing.T, sqlSecretStore *secretsKVStoreSQL, ctx context.Context, orgId int64, namespace1 string, typ string) {
+func validateSqlSecretWasDeleted(t *testing.T, sqlSecretStore *secretskvs.SecretsKVStoreSQL, ctx context.Context, orgId int64, namespace1 string, typ string) {
 	t.Helper()
 	res, err := sqlSecretStore.Keys(ctx, orgId, namespace1, typ)
 	require.NoError(t, err)
@@ -58,7 +80,7 @@ func validateSqlSecretWasDeleted(t *testing.T, sqlSecretStore *secretsKVStoreSQL
 }
 
 // validates that secrets should be on the plugin
-func validateSecretWasStoredInPlugin(t *testing.T, secretsStore SecretsKVStore, ctx context.Context, orgId int64, namespace1 string, typ string) {
+func validateSecretWasStoredInPlugin(t *testing.T, secretsStore secretskvs.SecretsKVStore, ctx context.Context, orgId int64, namespace1 string, typ string) {
 	t.Helper()
 	resPlugin, err := secretsStore.Keys(ctx, orgId, namespace1, typ)
 	require.NoError(t, err)
@@ -66,7 +88,7 @@ func validateSecretWasStoredInPlugin(t *testing.T, secretsStore SecretsKVStore,
 }
 
 // Set up services used in migration
-func setupTestMigrateToPluginService(t *testing.T) (*MigrateToPluginService, SecretsKVStore, *secretsKVStoreSQL) {
+func setupTestMigrateToPluginService(t *testing.T) (*MigrateToPluginService, secretskvs.SecretsKVStore, *secretskvs.SecretsKVStoreSQL) {
 	t.Helper()
 
 	rawCfg := `
@@ -77,12 +99,12 @@ func setupTestMigrateToPluginService(t *testing.T) (*MigrateToPluginService, Sec
 	require.NoError(t, err)
 	cfg := &setting.Cfg{Raw: raw}
 	// this would be the plugin - mocked at the moment
-	secretsStoreForPlugin := NewFakeSecretsKVStore()
+	secretsStoreForPlugin := secretskvs.NewFakeSecretsKVStore()
 
 	// this is to init the sql secret store inside the migration
 	sqlStore := sqlstore.InitTestDB(t)
 	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-	manager := NewFakeSecretsPluginManager(t, false)
+	manager := secretskvs.NewFakeSecretsPluginManager(t, false)
 	migratorService := ProvideMigrateToPluginService(
 		secretsStoreForPlugin,
 		cfg,
@@ -92,16 +114,39 @@ func setupTestMigrateToPluginService(t *testing.T) (*MigrateToPluginService, Sec
 		manager,
 	)
 
-	secretsSql := &secretsKVStoreSQL{
-		sqlStore:       sqlStore,
-		secretsService: secretsService,
-		log:            log.New("test.logger"),
-		decryptionCache: decryptionCache{
-			cache: make(map[int64]cachedDecrypted),
-		},
-	}
+	secretsSql := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
 
 	err = secretsStoreForPlugin.SetFallback(secretsSql)
 	require.NoError(t, err)
 	return migratorService, secretsStoreForPlugin, secretsSql
 }
+
+func setupTestMigratorServiceWithDeletionError(
+	t *testing.T,
+	secretskv secretskvs.SecretsKVStore,
+	sqlStore sqlstore.Store,
+	kvstore kvstore.KVStore,
+) *MigrateToPluginService {
+	t.Helper()
+	secretskvs.ResetPlugin()
+	cfg := secretskvs.SetupTestConfig(t)
+	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
+	manager := secretskvs.NewFakeSecretsPluginManager(t, false)
+	migratorService := ProvideMigrateToPluginService(
+		secretskv,
+		cfg,
+		sqlStore,
+		secretsService,
+		kvstore,
+		manager,
+	)
+	fallback := secretskvs.NewFakeSecretsKVStore()
+	var orgId int64 = 1
+	str := "random string"
+	err := fallback.Set(context.Background(), orgId, str, str, "bogus")
+	require.NoError(t, err)
+	fallback.DeletionError(true)
+	err = secretskv.SetFallback(fallback)
+	require.NoError(t, err)
+	return migratorService
+}

pkg/services/secrets/kvstore/model.go

@@ -7,6 +7,7 @@ import (
 const (
 	QuitOnPluginStartupFailureKey = "quit_on_secrets_plugin_startup_failure"
 	PluginNamespace               = "secretsmanagerplugin"
+	DataSourceSecretType          = "datasource"
 )
 
 // Item stored in k/v store.

pkg/services/secrets/kvstore/plugin.go

@@ -191,10 +191,10 @@ func updateFatalFlag(ctx context.Context, skv secretsKVStorePlugin) {
 	fatalFlagOnce.Do(func() {
 		skv.log.Debug("Updating plugin startup error fatal flag")
 		var err error
-		if isFatal, _ := isPluginStartupErrorFatal(ctx, skv.kvstore); !isFatal && skv.backwardsCompatibilityDisabled {
-			err = setPluginStartupErrorFatal(ctx, skv.kvstore, true)
+		if isFatal, _ := IsPluginStartupErrorFatal(ctx, skv.kvstore); !isFatal && skv.backwardsCompatibilityDisabled {
+			err = SetPluginStartupErrorFatal(ctx, skv.kvstore, true)
 		} else if isFatal && !skv.backwardsCompatibilityDisabled {
-			err = setPluginStartupErrorFatal(ctx, skv.kvstore, false)
+			err = SetPluginStartupErrorFatal(ctx, skv.kvstore, false)
 		}
 		if err != nil {
 			skv.log.Error("failed to set plugin error fatal flag", err.Error())
@@ -210,7 +210,7 @@ func GetNamespacedKVStore(kv kvstore.KVStore) *kvstore.NamespacedKVStore {
 	return kvstore.WithNamespace(kv, kvstore.AllOrganizations, PluginNamespace)
 }
 
-func isPluginStartupErrorFatal(ctx context.Context, kvstore *kvstore.NamespacedKVStore) (bool, error) {
+func IsPluginStartupErrorFatal(ctx context.Context, kvstore *kvstore.NamespacedKVStore) (bool, error) {
 	_, exists, err := kvstore.Get(ctx, QuitOnPluginStartupFailureKey)
 	if err != nil {
 		return false, errors.New(fmt.Sprint("error retrieving key ", QuitOnPluginStartupFailureKey, " from kvstore. error: ", err.Error()))
@@ -218,7 +218,7 @@ func isPluginStartupErrorFatal(ctx context.Context, kvstore *kvstore.NamespacedK
 	return exists, nil
 }
 
-func setPluginStartupErrorFatal(ctx context.Context, kvstore *kvstore.NamespacedKVStore, isFatal bool) error {
+func SetPluginStartupErrorFatal(ctx context.Context, kvstore *kvstore.NamespacedKVStore, isFatal bool) error {
 	if !isFatal {
 		return kvstore.Del(ctx, QuitOnPluginStartupFailureKey)
 	}
@@ -237,7 +237,7 @@ func EvaluateRemoteSecretsPlugin(mg plugins.SecretsPluginManager, cfg *setting.C
 	return nil
 }
 
-func startAndReturnPlugin(mg plugins.SecretsPluginManager, ctx context.Context) (smp.SecretsManagerPlugin, error) {
+func StartAndReturnPlugin(mg plugins.SecretsPluginManager, ctx context.Context) (smp.SecretsManagerPlugin, error) {
 	var err error
 	startupOnce.Do(func() {
 		err = mg.SecretsManager().Start(ctx)

pkg/services/secrets/kvstore/plugin_fatal_crash_test.go

@@ -1,184 +0,0 @@
-package kvstore
-
-import (
-	"context"
-	"errors"
-	"sync"
-	"testing"
-
-	"github.com/grafana/grafana/pkg/infra/kvstore"
-	"github.com/grafana/grafana/pkg/plugins"
-	"github.com/grafana/grafana/pkg/plugins/backendplugin/secretsmanagerplugin"
-	"github.com/grafana/grafana/pkg/services/secrets/fakes"
-	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
-	"github.com/grafana/grafana/pkg/services/sqlstore"
-	"github.com/grafana/grafana/pkg/services/sqlstore/mockstore"
-	"github.com/grafana/grafana/pkg/setting"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"gopkg.in/ini.v1"
-)
-
-// Set fatal flag to true, then simulate a plugin start failure
-// Should result in an error from the secret store provider
-func TestFatalPluginErr_PluginFailsToStartWithFatalFlagSet(t *testing.T) {
-	p, err := setupFatalCrashTest(t, true, true, false)
-	assert.Error(t, err)
-	assert.Equal(t, "mocked failed to start", err.Error())
-	assert.Nil(t, p.secretsKVStore)
-}
-
-// Set fatal flag to false, then simulate a plugin start failure
-// Should result in the secret store provider returning the sql impl
-func TestFatalPluginErr_PluginFailsToStartWithFatalFlagNotSet(t *testing.T) {
-	p, err := setupFatalCrashTest(t, true, false, false)
-	assert.NoError(t, err)
-	require.IsType(t, &CachedKVStore{}, p.secretsKVStore)
-
-	cachedKv, _ := p.secretsKVStore.(*CachedKVStore)
-	assert.IsType(t, &secretsKVStoreSQL{}, cachedKv.GetUnwrappedStore())
-}
-
-// With fatal flag not set, store a secret in the plugin while backwards compatibility is disabled
-// Should result in the fatal flag going from unset -> set to true
-func TestFatalPluginErr_FatalFlagGetsSetWithBackwardsCompatDisabled(t *testing.T) {
-	p, err := setupFatalCrashTest(t, false, false, true)
-	assert.NoError(t, err)
-	require.NotNil(t, p.secretsKVStore)
-
-	err = p.secretsKVStore.Set(context.Background(), 0, "datasource", "postgres", "my secret")
-	assert.NoError(t, err)
-
-	isFatal, err := isPluginStartupErrorFatal(context.Background(), GetNamespacedKVStore(p.kvstore))
-	assert.NoError(t, err)
-	assert.True(t, isFatal)
-}
-
-// With fatal flag set, retrieve a secret from the plugin while backwards compatibility is enabled
-// Should result in the fatal flag going from set to true -> unset
-func TestFatalPluginErr_FatalFlagGetsUnSetWithBackwardsCompatEnabled(t *testing.T) {
-	p, err := setupFatalCrashTest(t, false, true, false)
-	assert.NoError(t, err)
-	require.NotNil(t, p.secretsKVStore)
-
-	// setup - store secret and manually bypassing the remote plugin impl
-	_, err = p.pluginManager.SecretsManager().SecretsManager.SetSecret(context.Background(), &secretsmanagerplugin.SetSecretRequest{
-		KeyDescriptor: &secretsmanagerplugin.Key{
-			OrgId:     0,
-			Namespace: "postgres",
-			Type:      "datasource",
-		},
-		Value: "bogus",
-	})
-	assert.NoError(t, err)
-
-	// retrieve the secret and check values
-	val, exists, err := p.secretsKVStore.Get(context.Background(), 0, "postgres", "datasource")
-	assert.NoError(t, err)
-	assert.NotNil(t, val)
-	assert.True(t, exists)
-
-	isFatal, err := isPluginStartupErrorFatal(context.Background(), GetNamespacedKVStore(p.kvstore))
-	assert.NoError(t, err)
-	assert.False(t, isFatal)
-}
-
-// With fatal flag unset, do a migration with backwards compatibility disabled. When unified secrets are deleted, return an error on the first deletion
-// Should result in the fatal flag remaining unset
-func TestFatalPluginErr_MigrationTestWithErrorDeletingUnifiedSecrets(t *testing.T) {
-	p, err := setupFatalCrashTest(t, false, false, true)
-	assert.NoError(t, err)
-
-	migration := setupTestMigratorServiceWithDeletionError(t, p.secretsKVStore, &mockstore.SQLStoreMock{
-		ExpectedError: errors.New("random error"),
-	}, p.kvstore)
-	err = migration.Migrate(context.Background())
-	assert.Error(t, err)
-	assert.Equal(t, "mocked del error", err.Error())
-
-	isFatal, err := isPluginStartupErrorFatal(context.Background(), GetNamespacedKVStore(p.kvstore))
-	assert.NoError(t, err)
-	assert.False(t, isFatal)
-}
-
-func setupFatalCrashTest(
-	t *testing.T,
-	shouldFailOnStart bool,
-	isPluginErrorFatal bool,
-	isBackwardsCompatDisabled bool,
-) (fatalCrashTestFields, error) {
-	t.Helper()
-	fatalFlagOnce = sync.Once{}
-	startupOnce = sync.Once{}
-	cfg := setupTestConfig(t)
-	sqlStore := sqlstore.InitTestDB(t)
-	secretService := fakes.FakeSecretsService{}
-	kvstore := kvstore.ProvideService(sqlStore)
-	if isPluginErrorFatal {
-		_ = setPluginStartupErrorFatal(context.Background(), GetNamespacedKVStore(kvstore), true)
-	}
-	features := NewFakeFeatureToggles(t, isBackwardsCompatDisabled)
-	manager := NewFakeSecretsPluginManager(t, shouldFailOnStart)
-	svc, err := ProvideService(sqlStore, secretService, manager, kvstore, features, cfg)
-	t.Cleanup(func() {
-		fatalFlagOnce = sync.Once{}
-	})
-	return fatalCrashTestFields{
-		secretsKVStore: svc,
-		pluginManager:  manager,
-		kvstore:        kvstore,
-		sqlStore:       sqlStore,
-	}, err
-}
-
-type fatalCrashTestFields struct {
-	secretsKVStore SecretsKVStore
-	pluginManager  plugins.SecretsPluginManager
-	kvstore        kvstore.KVStore
-	sqlStore       *sqlstore.SQLStore
-}
-
-func setupTestMigratorServiceWithDeletionError(
-	t *testing.T,
-	secretskv SecretsKVStore,
-	sqlStore sqlstore.Store,
-	kvstore kvstore.KVStore,
-) *MigrateToPluginService {
-	t.Helper()
-	fatalFlagOnce = sync.Once{}
-	startupOnce = sync.Once{}
-	cfg := setupTestConfig(t)
-	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-	manager := NewFakeSecretsPluginManager(t, false)
-	migratorService := ProvideMigrateToPluginService(
-		secretskv,
-		cfg,
-		sqlStore,
-		secretsService,
-		kvstore,
-		manager,
-	)
-	fallback := NewFakeSecretsKVStore()
-	var orgId int64 = 1
-	str := "random string"
-	fallback.store[Key{
-		OrgId:     orgId,
-		Type:      str,
-		Namespace: str,
-	}] = "bogus"
-	fallback.delError = true
-	err := secretskv.SetFallback(fallback)
-	require.NoError(t, err)
-	return migratorService
-}
-
-func setupTestConfig(t *testing.T) *setting.Cfg {
-	t.Helper()
-	rawCfg := `
-		[secrets]
-		use_plugin = true
-		`
-	raw, err := ini.Load([]byte(rawCfg))
-	require.NoError(t, err)
-	return &setting.Cfg{Raw: raw}
-}

pkg/services/secrets/kvstore/plugin_test.go

@@ -0,0 +1,74 @@
+package kvstore
+
+import (
+	"context"
+	"testing"
+
+	"github.com/grafana/grafana/pkg/plugins/backendplugin/secretsmanagerplugin"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Set fatal flag to true, then simulate a plugin start failure
+// Should result in an error from the secret store provider
+func TestFatalPluginErr_PluginFailsToStartWithFatalFlagSet(t *testing.T) {
+	p, err := SetupFatalCrashTest(t, true, true, false)
+	assert.Error(t, err)
+	assert.Equal(t, "mocked failed to start", err.Error())
+	assert.Nil(t, p.SecretsKVStore)
+}
+
+// Set fatal flag to false, then simulate a plugin start failure
+// Should result in the secret store provider returning the sql impl
+func TestFatalPluginErr_PluginFailsToStartWithFatalFlagNotSet(t *testing.T) {
+	p, err := SetupFatalCrashTest(t, true, false, false)
+	assert.NoError(t, err)
+	require.IsType(t, &CachedKVStore{}, p.SecretsKVStore)
+
+	cachedKv, _ := p.SecretsKVStore.(*CachedKVStore)
+	assert.IsType(t, &SecretsKVStoreSQL{}, cachedKv.GetUnwrappedStore())
+}
+
+// With fatal flag not set, store a secret in the plugin while backwards compatibility is disabled
+// Should result in the fatal flag going from unset -> set to true
+func TestFatalPluginErr_FatalFlagGetsSetWithBackwardsCompatDisabled(t *testing.T) {
+	p, err := SetupFatalCrashTest(t, false, false, true)
+	assert.NoError(t, err)
+	require.NotNil(t, p.SecretsKVStore)
+
+	err = p.SecretsKVStore.Set(context.Background(), 0, "datasource", "postgres", "my secret")
+	assert.NoError(t, err)
+
+	isFatal, err := IsPluginStartupErrorFatal(context.Background(), GetNamespacedKVStore(p.KVStore))
+	assert.NoError(t, err)
+	assert.True(t, isFatal)
+}
+
+// With fatal flag set, retrieve a secret from the plugin while backwards compatibility is enabled
+// Should result in the fatal flag going from set to true -> unset
+func TestFatalPluginErr_FatalFlagGetsUnSetWithBackwardsCompatEnabled(t *testing.T) {
+	p, err := SetupFatalCrashTest(t, false, true, false)
+	assert.NoError(t, err)
+	require.NotNil(t, p.SecretsKVStore)
+
+	// setup - store secret and manually bypassing the remote plugin impl
+	_, err = p.PluginManager.SecretsManager().SecretsManager.SetSecret(context.Background(), &secretsmanagerplugin.SetSecretRequest{
+		KeyDescriptor: &secretsmanagerplugin.Key{
+			OrgId:     0,
+			Namespace: "postgres",
+			Type:      "datasource",
+		},
+		Value: "bogus",
+	})
+	assert.NoError(t, err)
+
+	// retrieve the secret and check values
+	val, exists, err := p.SecretsKVStore.Get(context.Background(), 0, "postgres", "datasource")
+	assert.NoError(t, err)
+	assert.NotNil(t, val)
+	assert.True(t, exists)
+
+	isFatal, err := IsPluginStartupErrorFatal(context.Background(), GetNamespacedKVStore(p.KVStore))
+	assert.NoError(t, err)
+	assert.False(t, isFatal)
+}

pkg/services/secrets/kvstore/sql.go

@@ -12,8 +12,8 @@ import (
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 )
 
-// secretsKVStoreSQL provides a key/value store backed by the Grafana database
-type secretsKVStoreSQL struct {
+// SecretsKVStoreSQL provides a key/value store backed by the Grafana database
+type SecretsKVStoreSQL struct {
 	log             log.Logger
 	sqlStore        sqlstore.Store
 	secretsService  secrets.Service
@@ -35,8 +35,19 @@ var (
 	errFallbackNotAllowed = errors.New("fallback not allowed for sql secret store")
 )
 
+func NewSQLSecretsKVStore(sqlStore sqlstore.Store, secretsService secrets.Service, logger log.Logger) *SecretsKVStoreSQL {
+	return &SecretsKVStoreSQL{
+		sqlStore:       sqlStore,
+		secretsService: secretsService,
+		log:            logger,
+		decryptionCache: decryptionCache{
+			cache: make(map[int64]cachedDecrypted),
+		},
+	}
+}
+
 // Get an item from the store
-func (kv *secretsKVStoreSQL) Get(ctx context.Context, orgId int64, namespace string, typ string) (string, bool, error) {
+func (kv *SecretsKVStoreSQL) Get(ctx context.Context, orgId int64, namespace string, typ string) (string, bool, error) {
 	item := Item{
 		OrgId:     &orgId,
 		Namespace: &namespace,
@@ -72,7 +83,7 @@ func (kv *secretsKVStoreSQL) Get(ctx context.Context, orgId int64, namespace str
 }
 
 // Set an item in the store
-func (kv *secretsKVStoreSQL) Set(ctx context.Context, orgId int64, namespace string, typ string, value string) error {
+func (kv *SecretsKVStoreSQL) Set(ctx context.Context, orgId int64, namespace string, typ string, value string) error {
 	encryptedValue, err := kv.secretsService.Encrypt(ctx, []byte(value), secrets.WithoutScope())
 	if err != nil {
 		kv.log.Error("error encrypting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
@@ -130,7 +141,7 @@ func (kv *secretsKVStoreSQL) Set(ctx context.Context, orgId int64, namespace str
 }
 
 // Del deletes an item from the store.
-func (kv *secretsKVStoreSQL) Del(ctx context.Context, orgId int64, namespace string, typ string) error {
+func (kv *SecretsKVStoreSQL) Del(ctx context.Context, orgId int64, namespace string, typ string) error {
 	err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
 		item := Item{
 			OrgId:     &orgId,
@@ -164,7 +175,7 @@ func (kv *secretsKVStoreSQL) Del(ctx context.Context, orgId int64, namespace str
 
 // Keys get all keys for a given namespace. To query for all
 // organizations the constant 'kvstore.AllOrganizations' can be passed as orgId.
-func (kv *secretsKVStoreSQL) Keys(ctx context.Context, orgId int64, namespace string, typ string) ([]Key, error) {
+func (kv *SecretsKVStoreSQL) Keys(ctx context.Context, orgId int64, namespace string, typ string) ([]Key, error) {
 	var keys []Key
 	err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
 		query := dbSession.Where("namespace = ?", namespace).And("type = ?", typ)
@@ -177,7 +188,7 @@ func (kv *secretsKVStoreSQL) Keys(ctx context.Context, orgId int64, namespace st
 }
 
 // Rename an item in the store
-func (kv *secretsKVStoreSQL) Rename(ctx context.Context, orgId int64, namespace string, typ string, newNamespace string) error {
+func (kv *SecretsKVStoreSQL) Rename(ctx context.Context, orgId int64, namespace string, typ string, newNamespace string) error {
 	return kv.sqlStore.WithTransactionalDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
 		item := Item{
 			OrgId:     &orgId,
@@ -211,7 +222,7 @@ func (kv *secretsKVStoreSQL) Rename(ctx context.Context, orgId int64, namespace
 
 // GetAll this returns all the secrets stored in the database. This is not part of the kvstore interface as we
 // only need it for migration from sql to plugin at this moment
-func (kv *secretsKVStoreSQL) GetAll(ctx context.Context) ([]Item, error) {
+func (kv *SecretsKVStoreSQL) GetAll(ctx context.Context) ([]Item, error) {
 	var items []Item
 	err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
 		return dbSession.Find(&items)
@@ -233,15 +244,15 @@ func (kv *secretsKVStoreSQL) GetAll(ctx context.Context) ([]Item, error) {
 	return items, err
 }
 
-func (kv *secretsKVStoreSQL) Fallback() SecretsKVStore {
+func (kv *SecretsKVStoreSQL) Fallback() SecretsKVStore {
 	return nil
 }
 
-func (kv *secretsKVStoreSQL) SetFallback(_ SecretsKVStore) error {
+func (kv *SecretsKVStoreSQL) SetFallback(_ SecretsKVStore) error {
 	return errFallbackNotAllowed
 }
 
-func (kv *secretsKVStoreSQL) getDecryptedValue(ctx context.Context, item Item) ([]byte, error) {
+func (kv *SecretsKVStoreSQL) getDecryptedValue(ctx context.Context, item Item) ([]byte, error) {
 	kv.decryptionCache.Lock()
 	defer kv.decryptionCache.Unlock()
 	var decryptedValue []byte

pkg/services/secrets/kvstore/sql_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 
 	"github.com/grafana/grafana/pkg/infra/log"
-	"github.com/grafana/grafana/pkg/services/secrets/database"
+	"github.com/grafana/grafana/pkg/services/secrets/fakes"
 	"github.com/grafana/grafana/pkg/services/secrets/manager"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/stretchr/testify/assert"
@@ -24,27 +24,10 @@ func (t *TestCase) Value() string {
 	return fmt.Sprintf("%d:%s:%s:%d", t.OrgId, t.Namespace, t.Type, t.Revision)
 }
 
-func setupTestService(t *testing.T) *secretsKVStoreSQL {
-	t.Helper()
-
-	sqlStore := sqlstore.InitTestDB(t)
-	store := database.ProvideSecretsStore(sqlstore.InitTestDB(t))
-	secretsService := manager.SetupTestService(t, store)
-
-	kv := &secretsKVStoreSQL{
-		sqlStore:       sqlStore,
-		log:            log.New("secrets.kvstore"),
-		secretsService: secretsService,
-		decryptionCache: decryptionCache{
-			cache: make(map[int64]cachedDecrypted),
-		},
-	}
-
-	return kv
-}
-
 func TestSecretsKVStoreSQL(t *testing.T) {
-	kv := setupTestService(t)
+	sqlStore := sqlstore.InitTestDB(t)
+	secretsService := manager.SetupTestService(t, fakes.NewFakeSecretsStore())
+	kv := NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
 
 	ctx := context.Background()
 
@@ -175,7 +158,9 @@ func TestSecretsKVStoreSQL(t *testing.T) {
 	})
 
 	t.Run("listing existing keys", func(t *testing.T) {
-		kv := setupTestService(t)
+		sqlStore := sqlstore.InitTestDB(t)
+		secretsService := manager.SetupTestService(t, fakes.NewFakeSecretsStore())
+		kv := NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
 
 		ctx := context.Background()
 
@@ -248,7 +233,9 @@ func TestSecretsKVStoreSQL(t *testing.T) {
 	})
 
 	t.Run("getting all secrets", func(t *testing.T) {
-		kv := setupTestService(t)
+		sqlStore := sqlstore.InitTestDB(t)
+		secretsService := manager.SetupTestService(t, fakes.NewFakeSecretsStore())
+		kv := NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
 
 		ctx := context.Background()
 

pkg/services/secrets/kvstore/test_helpers.go

@@ -3,38 +3,22 @@ package kvstore
 import (
 	"context"
 	"errors"
+	"sync"
 	"testing"
 
-	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/kvstore"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/plugins/backendplugin"
 	"github.com/grafana/grafana/pkg/plugins/backendplugin/secretsmanagerplugin"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
-	"github.com/grafana/grafana/pkg/services/secrets/database"
-	"github.com/grafana/grafana/pkg/services/secrets/manager"
+	"github.com/grafana/grafana/pkg/services/secrets/fakes"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc"
+	"gopkg.in/ini.v1"
 )
 
-func SetupTestService(t *testing.T) SecretsKVStore {
-	t.Helper()
-
-	sqlStore := sqlstore.InitTestDB(t)
-	store := database.ProvideSecretsStore(sqlstore.InitTestDB(t))
-	secretsService := manager.SetupTestService(t, store)
-
-	kv := &secretsKVStoreSQL{
-		sqlStore:       sqlStore,
-		log:            log.New("secrets.kvstore"),
-		secretsService: secretsService,
-		decryptionCache: decryptionCache{
-			cache: make(map[int64]cachedDecrypted),
-		},
-	}
-
-	return kv
-}
-
 // In memory kv store used for testing
 type FakeSecretsKVStore struct {
 	store    map[Key]string
@@ -46,6 +30,10 @@ func NewFakeSecretsKVStore() *FakeSecretsKVStore {
 	return &FakeSecretsKVStore{store: make(map[Key]string)}
 }
 
+func (f *FakeSecretsKVStore) DeletionError(shouldErr bool) {
+	f.delError = shouldErr
+}
+
 func (f *FakeSecretsKVStore) Get(ctx context.Context, orgId int64, namespace string, typ string) (string, bool, error) {
 	value := f.store[buildKey(orgId, namespace, typ)]
 	found := value != ""
@@ -247,3 +235,56 @@ func (pc *fakePluginClient) Start(_ context.Context) error {
 func (pc *fakePluginClient) Stop(_ context.Context) error {
 	return nil
 }
+
+func SetupFatalCrashTest(
+	t *testing.T,
+	shouldFailOnStart bool,
+	isPluginErrorFatal bool,
+	isBackwardsCompatDisabled bool,
+) (fatalCrashTestFields, error) {
+	t.Helper()
+	fatalFlagOnce = sync.Once{}
+	startupOnce = sync.Once{}
+	cfg := SetupTestConfig(t)
+	sqlStore := sqlstore.InitTestDB(t)
+	secretService := fakes.FakeSecretsService{}
+	kvstore := kvstore.ProvideService(sqlStore)
+	if isPluginErrorFatal {
+		_ = SetPluginStartupErrorFatal(context.Background(), GetNamespacedKVStore(kvstore), true)
+	}
+	features := NewFakeFeatureToggles(t, isBackwardsCompatDisabled)
+	manager := NewFakeSecretsPluginManager(t, shouldFailOnStart)
+	svc, err := ProvideService(sqlStore, secretService, manager, kvstore, features, cfg)
+	t.Cleanup(func() {
+		fatalFlagOnce = sync.Once{}
+	})
+	return fatalCrashTestFields{
+		SecretsKVStore: svc,
+		PluginManager:  manager,
+		KVStore:        kvstore,
+		SqlStore:       sqlStore,
+	}, err
+}
+
+type fatalCrashTestFields struct {
+	SecretsKVStore SecretsKVStore
+	PluginManager  plugins.SecretsPluginManager
+	KVStore        kvstore.KVStore
+	SqlStore       *sqlstore.SQLStore
+}
+
+func SetupTestConfig(t *testing.T) *setting.Cfg {
+	t.Helper()
+	rawCfg := `
+		[secrets]
+		use_plugin = true
+		`
+	raw, err := ini.Load([]byte(rawCfg))
+	require.NoError(t, err)
+	return &setting.Cfg{Raw: raw}
+}
+
+func ResetPlugin() {
+	fatalFlagOnce = sync.Once{}
+	startupOnce = sync.Once{}
+}