Secrets: Improve unified secrets migration and implement compatibility flag (#50463)

* Implement disableSecretsCompatibility flag

* Allow secret deletion right after migration

* Use dialect.Quote for secure_json_data on secret deletion

Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com>

* Set secure_json_data to NULL instead of empty json

* Run toggles_gen_test and use generated flag variable

* Add ID to delete data source secrets command on function call

Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com>

* Remove extra query to get datasource on secret deletion

* Fix linting issues with CHANGELOG.md

* Use empty json string when deleting secure json data

* Implement secret migration as a background process

* Refactor secret migration as a background service

* Refactor migration to be inside secret store

* Re-add secret deletion function removed on merge

* Try using transaction to fix db lock during tests

* Disable migration for pipeline debugging

* Try adding sleep to fix database lock

* Remove unecessary time sleep from migration

* Fix merge issue, replace models with datasources

* Try event listener approach

* Fix merge issue, replace models with datasources

* Fix linting issues with unchecked error

* Remove unecessary trainling new line

* Increase wait interval on background secret migration

* Rename secret store migration folder for consistency

* Convert background migration to blocking

* Fix number of arguments on server tests

* Check error value of secret migration provider

* Fix linting issue with method varaible

* Revert unintended change on background services

* Move secret migration service provider to wire.go

* Remove unecessary else from datasource service

* Move transaction inside loop on secret migration

* Remove unecessary GetServices function

* Remove unecessary interface after method removal

* Rename Run to Migrate on secret migration interface

* Rename secret migrations service variable on server

* Use MustBool on datasource secret migration

* Revert changes to GetDataSources

* Implement GetAllDataSources function

* Remove DeleteDataSourceSecrets function

* Move datasource secret migration to datasource service

* Remove unecessary properties from datasource secret migration

* Make DecryptLegacySecrets a private method

* Remove context canceled check on secret migrator

* Log error when fail to unmarshal datasource secret

* Add necessary fields to update command on migration

* Handle high availability on secret migration

* Use kvstore for datasource secret migration status

* Add error check for migration status set on kvstore

* Remove NewSecretMigrationService from server tests

* Use const for strings on datasource secrets migration

* Test all cases for datasources secret migrations

Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com>

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -59,6 +59,7 @@ export interface FeatureToggles {
   scenes?: boolean;
   useLegacyHeatmapPanel?: boolean;
   cloudMonitoringExperimentalUI?: boolean;
+  disableSecretsCompatibility?: boolean;
   logRequestsInstrumentedAsUnknown?: boolean;
   dataConnectionsConsole?: boolean;
   internationalization?: boolean;

pkg/events/events.go

@@ -55,6 +55,14 @@ type DataSourceDeleted struct {
 	OrgID     int64     `json:"org_id"`
 }
 
+type DataSourceSecretDeleted struct {
+	Timestamp time.Time `json:"timestamp"`
+	Name      string    `json:"name"`
+	ID        int64     `json:"id"`
+	UID       string    `json:"uid"`
+	OrgID     int64     `json:"org_id"`
+}
+
 type DataSourceCreated struct {
 	Timestamp time.Time `json:"timestamp"`
 	Name      string    `json:"name"`

pkg/server/server.go

@@ -24,6 +24,7 @@ import (
 	"github.com/grafana/grafana/pkg/login/social"
 	"github.com/grafana/grafana/pkg/registry"
 	"github.com/grafana/grafana/pkg/services/provisioning"
+	secretsMigrations "github.com/grafana/grafana/pkg/services/secrets/kvstore/migrations"
 
 	"github.com/grafana/grafana/pkg/setting"
 	"golang.org/x/sync/errgroup"
@@ -43,9 +44,10 @@ type Options struct {
 func New(opts Options, cfg *setting.Cfg, httpServer *api.HTTPServer, roleRegistry accesscontrol.RoleRegistry,
 	provisioningService provisioning.ProvisioningService, backgroundServiceProvider registry.BackgroundServiceRegistry,
 	usageStatsProvidersRegistry registry.UsageStatsProvidersRegistry, statsCollectorService *statscollector.Service,
+	secretMigrationService secretsMigrations.SecretMigrationService,
 ) (*Server, error) {
 	statsCollectorService.RegisterProviders(usageStatsProvidersRegistry.GetServices())
-	s, err := newServer(opts, cfg, httpServer, roleRegistry, provisioningService, backgroundServiceProvider)
+	s, err := newServer(opts, cfg, httpServer, roleRegistry, provisioningService, backgroundServiceProvider, secretMigrationService)
 	if err != nil {
 		return nil, err
 	}
@@ -59,25 +61,27 @@ func New(opts Options, cfg *setting.Cfg, httpServer *api.HTTPServer, roleRegistr
 
 func newServer(opts Options, cfg *setting.Cfg, httpServer *api.HTTPServer, roleRegistry accesscontrol.RoleRegistry,
 	provisioningService provisioning.ProvisioningService, backgroundServiceProvider registry.BackgroundServiceRegistry,
+	secretMigrationService secretsMigrations.SecretMigrationService,
 ) (*Server, error) {
 	rootCtx, shutdownFn := context.WithCancel(context.Background())
 	childRoutines, childCtx := errgroup.WithContext(rootCtx)
 
 	s := &Server{
-		context:             childCtx,
-		childRoutines:       childRoutines,
-		HTTPServer:          httpServer,
-		provisioningService: provisioningService,
-		roleRegistry:        roleRegistry,
-		shutdownFn:          shutdownFn,
-		shutdownFinished:    make(chan struct{}),
-		log:                 log.New("server"),
-		cfg:                 cfg,
-		pidFile:             opts.PidFile,
-		version:             opts.Version,
-		commit:              opts.Commit,
-		buildBranch:         opts.BuildBranch,
-		backgroundServices:  backgroundServiceProvider.GetServices(),
+		context:                childCtx,
+		childRoutines:          childRoutines,
+		HTTPServer:             httpServer,
+		provisioningService:    provisioningService,
+		roleRegistry:           roleRegistry,
+		shutdownFn:             shutdownFn,
+		shutdownFinished:       make(chan struct{}),
+		log:                    log.New("server"),
+		cfg:                    cfg,
+		pidFile:                opts.PidFile,
+		version:                opts.Version,
+		commit:                 opts.Commit,
+		buildBranch:            opts.BuildBranch,
+		backgroundServices:     backgroundServiceProvider.GetServices(),
+		secretMigrationService: secretMigrationService,
 	}
 
 	return s, nil
@@ -101,9 +105,10 @@ type Server struct {
 	buildBranch        string
 	backgroundServices []registry.BackgroundService
 
-	HTTPServer          *api.HTTPServer
-	roleRegistry        accesscontrol.RoleRegistry
-	provisioningService provisioning.ProvisioningService
+	HTTPServer             *api.HTTPServer
+	roleRegistry           accesscontrol.RoleRegistry
+	provisioningService    provisioning.ProvisioningService
+	secretMigrationService secretsMigrations.SecretMigrationService
 }
 
 // init initializes the server and its services.
@@ -128,6 +133,10 @@ func (s *Server) init() error {
 		return err
 	}
 
+	if err := s.secretMigrationService.Migrate(s.context); err != nil {
+		return err
+	}
+
 	return s.provisioningService.RunInitProvisioners(s.context)
 }
 

pkg/server/server_test.go

@@ -7,9 +7,12 @@ import (
 	"testing"
 	"time"
 
+	"github.com/grafana/grafana/pkg/infra/serverlock"
 	"github.com/grafana/grafana/pkg/registry"
 	"github.com/grafana/grafana/pkg/server/backgroundsvcs"
 	"github.com/grafana/grafana/pkg/services/accesscontrol/ossaccesscontrol"
+	"github.com/grafana/grafana/pkg/services/secrets/kvstore/migrations"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/stretchr/testify/require"
 )
@@ -47,7 +50,11 @@ func (s *testService) IsDisabled() bool {
 
 func testServer(t *testing.T, services ...registry.BackgroundService) *Server {
 	t.Helper()
-	s, err := newServer(Options{}, setting.NewCfg(), nil, &ossaccesscontrol.OSSAccessControlService{}, nil, backgroundsvcs.NewBackgroundServiceRegistry(services...))
+	serverLockService := serverlock.ProvideService(sqlstore.InitTestDB(t))
+	secretMigrationService := &migrations.SecretMigrationServiceImpl{
+		ServerLockService: serverLockService,
+	}
+	s, err := newServer(Options{}, setting.NewCfg(), nil, &ossaccesscontrol.OSSAccessControlService{}, nil, backgroundsvcs.NewBackgroundServiceRegistry(services...), secretMigrationService)
 	require.NoError(t, err)
 	// Required to skip configuration initialization that causes
 	// DI errors in this test.

pkg/server/wire.go

@@ -91,6 +91,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/secrets"
 	secretsDatabase "github.com/grafana/grafana/pkg/services/secrets/database"
 	secretsStore "github.com/grafana/grafana/pkg/services/secrets/kvstore"
+	secretsMigrations "github.com/grafana/grafana/pkg/services/secrets/kvstore/migrations"
 	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
 	secretsMigrator "github.com/grafana/grafana/pkg/services/secrets/migrator"
 	"github.com/grafana/grafana/pkg/services/serviceaccounts"
@@ -293,6 +294,9 @@ var wireBasicSet = wire.NewSet(
 	publicdashboardsApi.ProvideApi,
 	userimpl.ProvideService,
 	orgimpl.ProvideService,
+	datasourceservice.ProvideDataSourceMigrationService,
+	secretsMigrations.ProvideSecretMigrationService,
+	wire.Bind(new(secretsMigrations.SecretMigrationService), new(*secretsMigrations.SecretMigrationServiceImpl)),
 )
 
 var wireSet = wire.NewSet(

pkg/services/datasources/datasources.go

@@ -18,6 +18,9 @@ type DataSourceService interface {
 	// GetDataSources gets datasources.
 	GetDataSources(ctx context.Context, query *GetDataSourcesQuery) error
 
+	// GetAllDataSources gets all datasources.
+	GetAllDataSources(ctx context.Context, query *GetAllDataSourcesQuery) error
+
 	// GetDataSourcesByType gets datasources by type.
 	GetDataSourcesByType(ctx context.Context, query *GetDataSourcesByTypeQuery) error
 

pkg/services/datasources/fakes/fake_datasource_service.go

@@ -41,6 +41,11 @@ func (s *FakeDataSourceService) GetDataSources(ctx context.Context, query *datas
 	return nil
 }
 
+func (s *FakeDataSourceService) GetAllDataSources(ctx context.Context, query *datasources.GetAllDataSourcesQuery) error {
+	query.Result = s.DataSources
+	return nil
+}
+
 func (s *FakeDataSourceService) GetDataSourcesByType(ctx context.Context, query *datasources.GetDataSourcesByTypeQuery) error {
 	for _, datasource := range s.DataSources {
 		typeMatch := query.Type != "" && query.Type == datasource.Type

pkg/services/datasources/models.go

@@ -160,6 +160,10 @@ type GetDataSourcesQuery struct {
 	Result          []*DataSource
 }
 
+type GetAllDataSourcesQuery struct {
+	Result []*DataSource
+}
+
 type GetDataSourcesByTypeQuery struct {
 	Type   string
 	Result []*DataSource

pkg/services/datasources/service/datasource_service.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/httpclient"
+	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
 	"github.com/grafana/grafana/pkg/services/datasources"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
@@ -32,6 +33,7 @@ type Service struct {
 	features           featuremgmt.FeatureToggles
 	permissionsService accesscontrol.DatasourcePermissionsService
 	ac                 accesscontrol.AccessControl
+	logger             log.Logger
 
 	ptc proxyTransportCache
 }
@@ -61,6 +63,7 @@ func ProvideService(
 		features:           features,
 		permissionsService: datasourcePermissionsService,
 		ac:                 ac,
+		logger:             log.New("datasources"),
 	}
 
 	ac.RegisterScopeAttributeResolver(NewNameScopeResolver(store))
@@ -136,6 +139,10 @@ func (s *Service) GetDataSources(ctx context.Context, query *datasources.GetData
 	return s.SQLStore.GetDataSources(ctx, query)
 }
 
+func (s *Service) GetAllDataSources(ctx context.Context, query *datasources.GetAllDataSourcesQuery) error {
+	return s.SQLStore.GetAllDataSources(ctx, query)
+}
+
 func (s *Service) GetDataSourcesByType(ctx context.Context, query *datasources.GetDataSourcesByTypeQuery) error {
 	return s.SQLStore.GetDataSourcesByType(ctx, query)
 }
@@ -143,18 +150,21 @@ func (s *Service) GetDataSourcesByType(ctx context.Context, query *datasources.G
 func (s *Service) AddDataSource(ctx context.Context, cmd *datasources.AddDataSourceCommand) error {
 	return s.SQLStore.InTransaction(ctx, func(ctx context.Context) error {
 		var err error
-		// this is here for backwards compatibility
-		cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope())
-		if err != nil {
-			return err
-		}
 
-		secret, err := json.Marshal(cmd.SecureJsonData)
-		if err != nil {
-			return err
+		cmd.EncryptedSecureJsonData = make(map[string][]byte)
+		if !s.features.IsEnabled(featuremgmt.FlagDisableSecretsCompatibility) {
+			cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope())
+			if err != nil {
+				return err
+			}
 		}
 
 		cmd.UpdateSecretFn = func() error {
+			secret, err := json.Marshal(cmd.SecureJsonData)
+			if err != nil {
+				return err
+			}
+
 			return s.SecretsStore.Set(ctx, cmd.OrgId, cmd.Name, secretType, string(secret))
 		}
 
@@ -212,21 +222,22 @@ func (s *Service) UpdateDataSource(ctx context.Context, cmd *datasources.UpdateD
 			return err
 		}
 
-		secret, err := json.Marshal(cmd.SecureJsonData)
-		if err != nil {
-			return err
-		}
+		if cmd.OrgId > 0 && cmd.Name != "" {
+			cmd.UpdateSecretFn = func() error {
+				secret, err := json.Marshal(cmd.SecureJsonData)
+				if err != nil {
+					return err
+				}
 
-		cmd.UpdateSecretFn = func() error {
-			var secretsErr error
-			if query.Result.Name != cmd.Name {
-				secretsErr = s.SecretsStore.Rename(ctx, cmd.OrgId, query.Result.Name, secretType, cmd.Name)
-			}
-			if secretsErr != nil {
-				return secretsErr
-			}
+				if query.Result.Name != cmd.Name {
+					err := s.SecretsStore.Rename(ctx, cmd.OrgId, query.Result.Name, secretType, cmd.Name)
+					if err != nil {
+						return err
+					}
+				}
 
-			return s.SecretsStore.Set(ctx, cmd.OrgId, cmd.Name, secretType, string(secret))
+				return s.SecretsStore.Set(ctx, cmd.OrgId, cmd.Name, secretType, string(secret))
+			}
 		}
 
 		return s.SQLStore.UpdateDataSource(ctx, cmd)
@@ -295,10 +306,13 @@ func (s *Service) DecryptedValues(ctx context.Context, ds *datasources.DataSourc
 
 	if exist {
 		err = json.Unmarshal([]byte(secret), &decryptedValues)
+		if err != nil {
+			s.logger.Debug("failed to unmarshal secret value, using legacy secrets", "err", err)
+		}
 	}
 
-	if (!exist || err != nil) && len(ds.SecureJsonData) > 0 {
-		decryptedValues, err = s.MigrateSecrets(ctx, ds)
+	if !exist || err != nil {
+		decryptedValues, err = s.decryptLegacySecrets(ctx, ds)
 		if err != nil {
 			return nil, err
 		}
@@ -307,7 +321,7 @@ func (s *Service) DecryptedValues(ctx context.Context, ds *datasources.DataSourc
 	return decryptedValues, nil
 }
 
-func (s *Service) MigrateSecrets(ctx context.Context, ds *datasources.DataSource) (map[string]string, error) {
+func (s *Service) decryptLegacySecrets(ctx context.Context, ds *datasources.DataSource) (map[string]string, error) {
 	secureJsonData := make(map[string]string)
 	for k, v := range ds.SecureJsonData {
 		decrypted, err := s.SecretsService.Decrypt(ctx, v)
@@ -316,14 +330,7 @@ func (s *Service) MigrateSecrets(ctx context.Context, ds *datasources.DataSource
 		}
 		secureJsonData[k] = string(decrypted)
 	}
-
-	jsonData, err := json.Marshal(secureJsonData)
-	if err != nil {
-		return nil, err
-	}
-
-	err = s.SecretsStore.Set(ctx, ds.OrgId, ds.Name, secretType, string(jsonData))
-	return secureJsonData, err
+	return secureJsonData, nil
 }
 
 func (s *Service) DecryptedValue(ctx context.Context, ds *datasources.DataSource, key string) (string, bool, error) {
@@ -564,10 +571,12 @@ func (s *Service) fillWithSecureJSONData(ctx context.Context, cmd *datasources.U
 		}
 	}
 
-	// this is here for backwards compatibility
-	cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope())
-	if err != nil {
-		return err
+	cmd.EncryptedSecureJsonData = make(map[string][]byte)
+	if !s.features.IsEnabled(featuremgmt.FlagDisableSecretsCompatibility) {
+		cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope())
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil

pkg/services/datasources/service/secrets_mig.go

@@ -0,0 +1,92 @@
+package service
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/infra/kvstore"
+	"github.com/grafana/grafana/pkg/services/datasources"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+)
+
+const (
+	secretMigrationStatusKey       = "secretMigrationStatus"
+	compatibleSecretMigrationValue = "compatible"
+	completeSecretMigrationValue   = "complete"
+)
+
+type DataSourceSecretMigrationService struct {
+	dataSourcesService datasources.DataSourceService
+	kvStore            *kvstore.NamespacedKVStore
+	features           featuremgmt.FeatureToggles
+}
+
+func ProvideDataSourceMigrationService(
+	dataSourcesService datasources.DataSourceService,
+	kvStore kvstore.KVStore,
+	features featuremgmt.FeatureToggles,
+) *DataSourceSecretMigrationService {
+	return &DataSourceSecretMigrationService{
+		dataSourcesService: dataSourcesService,
+		kvStore:            kvstore.WithNamespace(kvStore, 0, secretType),
+		features:           features,
+	}
+}
+
+func (s *DataSourceSecretMigrationService) Migrate(ctx context.Context) error {
+	migrationStatus, _, err := s.kvStore.Get(ctx, secretMigrationStatusKey)
+	if err != nil {
+		return err
+	}
+
+	disableSecretsCompatibility := s.features.IsEnabled(featuremgmt.FlagDisableSecretsCompatibility)
+	needCompatibility := migrationStatus != compatibleSecretMigrationValue && !disableSecretsCompatibility
+	needMigration := migrationStatus != completeSecretMigrationValue && disableSecretsCompatibility
+
+	if needCompatibility || needMigration {
+		query := &datasources.GetAllDataSourcesQuery{}
+		err := s.dataSourcesService.GetAllDataSources(ctx, query)
+		if err != nil {
+			return err
+		}
+
+		for _, ds := range query.Result {
+			secureJsonData, err := s.dataSourcesService.DecryptedValues(ctx, ds)
+			if err != nil {
+				return err
+			}
+
+			// Secrets are set by the update data source function if the SecureJsonData is set in the command
+			// Secrets are deleted by the update data source function if the disableSecretsCompatibility flag is enabled
+			err = s.dataSourcesService.UpdateDataSource(ctx, &datasources.UpdateDataSourceCommand{
+				Id:             ds.Id,
+				OrgId:          ds.OrgId,
+				Uid:            ds.Uid,
+				Name:           ds.Name,
+				JsonData:       ds.JsonData,
+				SecureJsonData: secureJsonData,
+
+				// These are needed by the SQL function due to UseBool and MustCols
+				IsDefault:       ds.IsDefault,
+				BasicAuth:       ds.BasicAuth,
+				WithCredentials: ds.WithCredentials,
+				ReadOnly:        ds.ReadOnly,
+				User:            ds.User,
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		if disableSecretsCompatibility {
+			err = s.kvStore.Set(ctx, secretMigrationStatusKey, completeSecretMigrationValue)
+		} else {
+			err = s.kvStore.Set(ctx, secretMigrationStatusKey, compatibleSecretMigrationValue)
+		}
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/services/datasources/service/secrets_mig_test.go

@@ -0,0 +1,340 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	"github.com/grafana/grafana/pkg/infra/kvstore"
+	acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
+	"github.com/grafana/grafana/pkg/services/datasources"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/services/secrets/fakes"
+	secretsStore "github.com/grafana/grafana/pkg/services/secrets/kvstore"
+	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/stretchr/testify/assert"
+)
+
+func SetupTestMigrationService(t *testing.T, sqlStore *sqlstore.SQLStore, kvStore kvstore.KVStore, secretsStore secretsStore.SecretsKVStore, compatibility bool) *DataSourceSecretMigrationService {
+	t.Helper()
+	cfg := &setting.Cfg{}
+	features := featuremgmt.WithFeatures()
+	if !compatibility {
+		features = featuremgmt.WithFeatures(featuremgmt.FlagDisableSecretsCompatibility, true)
+	}
+	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
+	dsService := ProvideService(sqlStore, secretsService, secretsStore, cfg, features, acmock.New().WithDisabled(), acmock.NewMockedPermissionsService())
+	migService := ProvideDataSourceMigrationService(dsService, kvStore, features)
+	return migService
+}
+
+func TestMigrate(t *testing.T) {
+	t.Run("should migrate from legacy to unified without compatibility", func(t *testing.T) {
+		sqlStore := sqlstore.InitTestDB(t)
+		kvStore := kvstore.ProvideService(sqlStore)
+		secretsStore := secretsStore.SetupTestService(t)
+		migService := SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, false)
+
+		dataSourceName := "Test"
+		dataSourceOrg := int64(1)
+
+		// Add test data source
+		err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
+			OrgId:  dataSourceOrg,
+			Name:   dataSourceName,
+			Type:   datasources.DS_MYSQL,
+			Access: datasources.DS_ACCESS_DIRECT,
+			Url:    "http://test",
+			EncryptedSecureJsonData: map[string][]byte{
+				"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
+			},
+		})
+		assert.NoError(t, err)
+
+		// Check if the secret json data was added
+		query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.NotEmpty(t, query.Result.SecureJsonData)
+
+		// Check if the migration status key is empty
+		value, exist, err := kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Check that the secret is not present on the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Run the migration
+		err = migService.Migrate(context.Background())
+		assert.NoError(t, err)
+
+		// Check if the secure json data was deleted
+		query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.Empty(t, query.Result.SecureJsonData)
+
+		// Check if the secret was added to the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, value)
+		assert.True(t, exist)
+
+		// Check if the migration status key was set
+		value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Equal(t, completeSecretMigrationValue, value)
+		assert.True(t, exist)
+	})
+
+	t.Run("should migrate from legacy to unified with compatibility", func(t *testing.T) {
+		sqlStore := sqlstore.InitTestDB(t)
+		kvStore := kvstore.ProvideService(sqlStore)
+		secretsStore := secretsStore.SetupTestService(t)
+		migService := SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, true)
+
+		dataSourceName := "Test"
+		dataSourceOrg := int64(1)
+
+		// Add test data source
+		err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
+			OrgId:  dataSourceOrg,
+			Name:   dataSourceName,
+			Type:   datasources.DS_MYSQL,
+			Access: datasources.DS_ACCESS_DIRECT,
+			Url:    "http://test",
+			EncryptedSecureJsonData: map[string][]byte{
+				"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
+			},
+		})
+		assert.NoError(t, err)
+
+		// Check if the secret json data was added
+		query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.NotEmpty(t, query.Result.SecureJsonData)
+
+		// Check if the migration status key is empty
+		value, exist, err := kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Check that the secret is not present on the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Run the migration
+		err = migService.Migrate(context.Background())
+		assert.NoError(t, err)
+
+		// Check if the secure json data was maintained for compatibility
+		query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.NotEmpty(t, query.Result.SecureJsonData)
+
+		// Check if the secret was added to the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, value)
+		assert.True(t, exist)
+
+		// Check if the migration status key was set
+		value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Equal(t, compatibleSecretMigrationValue, value)
+		assert.True(t, exist)
+	})
+
+	t.Run("should replicate from unified to legacy for compatibility", func(t *testing.T) {
+		sqlStore := sqlstore.InitTestDB(t)
+		kvStore := kvstore.ProvideService(sqlStore)
+		secretsStore := secretsStore.SetupTestService(t)
+		migService := SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, false)
+
+		dataSourceName := "Test"
+		dataSourceOrg := int64(1)
+
+		// Add test data source
+		err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
+			OrgId:  dataSourceOrg,
+			Name:   dataSourceName,
+			Type:   datasources.DS_MYSQL,
+			Access: datasources.DS_ACCESS_DIRECT,
+			Url:    "http://test",
+			EncryptedSecureJsonData: map[string][]byte{
+				"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
+			},
+		})
+		assert.NoError(t, err)
+
+		// Check if the secret json data was added
+		query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.NotEmpty(t, query.Result.SecureJsonData)
+
+		// Check if the migration status key is empty
+		value, exist, err := kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Check that the secret is not present on the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Run the migration without compatibility
+		err = migService.Migrate(context.Background())
+		assert.NoError(t, err)
+
+		// Check if the secure json data was deleted
+		query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.Empty(t, query.Result.SecureJsonData)
+
+		// Check if the secret was added to the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, value)
+		assert.True(t, exist)
+
+		// Check if the migration status key was set
+		value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Equal(t, completeSecretMigrationValue, value)
+		assert.True(t, exist)
+
+		// Run the migration with compatibility
+		migService = SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, true)
+		err = migService.Migrate(context.Background())
+		assert.NoError(t, err)
+
+		// Check if the secure json data was re-added for compatibility
+		query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.NotEmpty(t, query.Result.SecureJsonData)
+
+		// Check if the secret was added to the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, value)
+		assert.True(t, exist)
+
+		// Check if the migration status key was set
+		value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Equal(t, compatibleSecretMigrationValue, value)
+		assert.True(t, exist)
+	})
+
+	t.Run("should delete from legacy to remove compatibility", func(t *testing.T) {
+		sqlStore := sqlstore.InitTestDB(t)
+		kvStore := kvstore.ProvideService(sqlStore)
+		secretsStore := secretsStore.SetupTestService(t)
+		migService := SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, true)
+
+		dataSourceName := "Test"
+		dataSourceOrg := int64(1)
+
+		// Add test data source
+		err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
+			OrgId:  dataSourceOrg,
+			Name:   dataSourceName,
+			Type:   datasources.DS_MYSQL,
+			Access: datasources.DS_ACCESS_DIRECT,
+			Url:    "http://test",
+			EncryptedSecureJsonData: map[string][]byte{
+				"password": []byte("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"),
+			},
+		})
+		assert.NoError(t, err)
+
+		// Check if the secret json data was added
+		query := &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.NotEmpty(t, query.Result.SecureJsonData)
+
+		// Check if the migration status key is empty
+		value, exist, err := kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Check that the secret is not present on the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
+		assert.NoError(t, err)
+		assert.Empty(t, value)
+		assert.False(t, exist)
+
+		// Run the migration with compatibility
+		err = migService.Migrate(context.Background())
+		assert.NoError(t, err)
+
+		// Check if the secure json data was maintained for compatibility
+		query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.NotEmpty(t, query.Result.SecureJsonData)
+
+		// Check if the secret was added to the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, value)
+		assert.True(t, exist)
+
+		// Check if the migration status key was set
+		value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Equal(t, compatibleSecretMigrationValue, value)
+		assert.True(t, exist)
+
+		// Run the migration without compatibility
+		migService = SetupTestMigrationService(t, sqlStore, kvStore, secretsStore, false)
+		err = migService.Migrate(context.Background())
+		assert.NoError(t, err)
+
+		// Check if the secure json data was deleted
+		query = &datasources.GetDataSourceQuery{OrgId: dataSourceOrg, Name: dataSourceName}
+		err = sqlStore.GetDataSource(context.Background(), query)
+		assert.NoError(t, err)
+		assert.NotNil(t, query.Result)
+		assert.Empty(t, query.Result.SecureJsonData)
+
+		// Check if the secret was added to the secret store
+		value, exist, err = secretsStore.Get(context.Background(), dataSourceOrg, dataSourceName, secretType)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, value)
+		assert.True(t, exist)
+
+		// Check if the migration status key was set
+		value, exist, err = kvStore.Get(context.Background(), 0, secretType, secretMigrationStatusKey)
+		assert.NoError(t, err)
+		assert.Equal(t, completeSecretMigrationValue, value)
+		assert.True(t, exist)
+	})
+}

pkg/services/featuremgmt/registry.go

@@ -244,6 +244,12 @@ var (
 			State:        FeatureStateAlpha,
 			FrontendOnly: true,
 		},
+		{
+			Name:            "disableSecretsCompatibility",
+			Description:     "Disable duplicated secret storage in legacy tables",
+			State:           FeatureStateAlpha,
+			RequiresRestart: true,
+		},
 		{
 			Name:        "logRequestsInstrumentedAsUnknown",
 			Description: "Logs the path for requests that are instrumented as unknown",

pkg/services/featuremgmt/toggles_gen.go

@@ -179,6 +179,10 @@ const (
 	// Use grafana-experimental UI in Cloud Monitoring
 	FlagCloudMonitoringExperimentalUI = "cloudMonitoringExperimentalUI"
 
+	// FlagDisableSecretsCompatibility
+	// Disable duplicated secret storage in legacy tables
+	FlagDisableSecretsCompatibility = "disableSecretsCompatibility"
+
 	// FlagLogRequestsInstrumentedAsUnknown
 	// Logs the path for requests that are instrumented as unknown
 	FlagLogRequestsInstrumentedAsUnknown = "logRequestsInstrumentedAsUnknown"

pkg/services/secrets/kvstore/migrations/migrator.go

@@ -0,0 +1,51 @@
+package migrations
+
+import (
+	"context"
+	"reflect"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/serverlock"
+	datasources "github.com/grafana/grafana/pkg/services/datasources/service"
+)
+
+var logger = log.New("secret.migration")
+
+// SecretMigrationService is used to migrate legacy secrets to new unified secrets.
+type SecretMigrationService interface {
+	Migrate(ctx context.Context) error
+}
+
+type SecretMigrationServiceImpl struct {
+	Services          []SecretMigrationService
+	ServerLockService *serverlock.ServerLockService
+}
+
+func ProvideSecretMigrationService(
+	serverLockService *serverlock.ServerLockService,
+	dataSourceSecretMigrationService *datasources.DataSourceSecretMigrationService,
+) *SecretMigrationServiceImpl {
+	return &SecretMigrationServiceImpl{
+		ServerLockService: serverLockService,
+		Services: []SecretMigrationService{
+			dataSourceSecretMigrationService,
+		},
+	}
+}
+
+// Run migration services. This will block until all services have exited.
+func (s *SecretMigrationServiceImpl) Migrate(ctx context.Context) error {
+	// Start migration services.
+	return s.ServerLockService.LockAndExecute(ctx, "migrate secrets to unified secrets", time.Minute*10, func(context.Context) {
+		for _, service := range s.Services {
+			serviceName := reflect.TypeOf(service).String()
+			logger.Debug("Starting secret migration service", "service", serviceName)
+			err := service.Migrate(ctx)
+			if err != nil {
+				logger.Error("Stopped secret migration service", "service", serviceName, "reason", err)
+			}
+			logger.Debug("Finished secret migration service", "service", serviceName)
+		}
+	})
+}

pkg/services/sqlstore/datasource.go

@@ -61,6 +61,13 @@ func (ss *SQLStore) GetDataSources(ctx context.Context, query *datasources.GetDa
 	})
 }
 
+func (ss *SQLStore) GetAllDataSources(ctx context.Context, query *datasources.GetAllDataSourcesQuery) error {
+	return ss.WithDbSession(ctx, func(sess *DBSession) error {
+		query.Result = make([]*datasources.DataSource, 0)
+		return sess.Asc("name").Find(&query.Result)
+	})
+}
+
 // GetDataSourcesByType returns all datasources for a given type or an error if the specified type is an empty string
 func (ss *SQLStore) GetDataSourcesByType(ctx context.Context, query *datasources.GetDataSourcesByTypeQuery) error {
 	if query.Type == "" {
@@ -255,6 +262,9 @@ func (ss *SQLStore) UpdateDataSource(ctx context.Context, cmd *datasources.Updat
 		sess.MustCols("password")
 		sess.MustCols("basic_auth_password")
 		sess.MustCols("user")
+		// Make sure secure json data is zeroed out if empty. We do this as we want to migrate secrets from
+		// secure json data to the unified secrets table.
+		sess.MustCols("secure_json_data")
 
 		var updateSession *xorm.Session
 		if cmd.Version != 0 {