Secrets: Improve unified secrets migration and implement compatibility flag (#50463)

* Implement disableSecretsCompatibility flag

* Allow secret deletion right after migration

* Use dialect.Quote for secure_json_data on secret deletion

Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com>

* Set secure_json_data to NULL instead of empty json

* Run toggles_gen_test and use generated flag variable

* Add ID to delete data source secrets command on function call

Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com>

* Remove extra query to get datasource on secret deletion

* Fix linting issues with CHANGELOG.md

* Use empty json string when deleting secure json data

* Implement secret migration as a background process

* Refactor secret migration as a background service

* Refactor migration to be inside secret store

* Re-add secret deletion function removed on merge

* Try using transaction to fix db lock during tests

* Disable migration for pipeline debugging

* Try adding sleep to fix database lock

* Remove unecessary time sleep from migration

* Fix merge issue, replace models with datasources

* Try event listener approach

* Fix merge issue, replace models with datasources

* Fix linting issues with unchecked error

* Remove unecessary trainling new line

* Increase wait interval on background secret migration

* Rename secret store migration folder for consistency

* Convert background migration to blocking

* Fix number of arguments on server tests

* Check error value of secret migration provider

* Fix linting issue with method varaible

* Revert unintended change on background services

* Move secret migration service provider to wire.go

* Remove unecessary else from datasource service

* Move transaction inside loop on secret migration

* Remove unecessary GetServices function

* Remove unecessary interface after method removal

* Rename Run to Migrate on secret migration interface

* Rename secret migrations service variable on server

* Use MustBool on datasource secret migration

* Revert changes to GetDataSources

* Implement GetAllDataSources function

* Remove DeleteDataSourceSecrets function

* Move datasource secret migration to datasource service

* Remove unecessary properties from datasource secret migration

* Make DecryptLegacySecrets a private method

* Remove context canceled check on secret migrator

* Log error when fail to unmarshal datasource secret

* Add necessary fields to update command on migration

* Handle high availability on secret migration

* Use kvstore for datasource secret migration status

* Add error check for migration status set on kvstore

* Remove NewSecretMigrationService from server tests

* Use const for strings on datasource secrets migration

* Test all cases for datasources secret migrations

Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com>

pkg/services/secrets/kvstore/migrations/migrator.go

@@ -0,0 +1,51 @@
+package migrations
+
+import (
+	"context"
+	"reflect"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/serverlock"
+	datasources "github.com/grafana/grafana/pkg/services/datasources/service"
+)
+
+var logger = log.New("secret.migration")
+
+// SecretMigrationService is used to migrate legacy secrets to new unified secrets.
+type SecretMigrationService interface {
+	Migrate(ctx context.Context) error
+}
+
+type SecretMigrationServiceImpl struct {
+	Services          []SecretMigrationService
+	ServerLockService *serverlock.ServerLockService
+}
+
+func ProvideSecretMigrationService(
+	serverLockService *serverlock.ServerLockService,
+	dataSourceSecretMigrationService *datasources.DataSourceSecretMigrationService,
+) *SecretMigrationServiceImpl {
+	return &SecretMigrationServiceImpl{
+		ServerLockService: serverLockService,
+		Services: []SecretMigrationService{
+			dataSourceSecretMigrationService,
+		},
+	}
+}
+
+// Run migration services. This will block until all services have exited.
+func (s *SecretMigrationServiceImpl) Migrate(ctx context.Context) error {
+	// Start migration services.
+	return s.ServerLockService.LockAndExecute(ctx, "migrate secrets to unified secrets", time.Minute*10, func(context.Context) {
+		for _, service := range s.Services {
+			serviceName := reflect.TypeOf(service).String()
+			logger.Debug("Starting secret migration service", "service", serviceName)
+			err := service.Migrate(ctx)
+			if err != nil {
+				logger.Error("Stopped secret migration service", "service", serviceName, "reason", err)
+			}
+			logger.Debug("Finished secret migration service", "service", serviceName)
+		}
+	})
+}