Secrets: Implement basic unified secret store service (#45804)

* wip: Implement kvstore for secrets

* wip: Refactor kvstore for secrets

* wip: Add format key function to secrets kvstore sql

* wip: Add migration for secrets kvstore

* Remove unused Key field from secrets kvstore

* Remove secret values from debug logs

* Integrate unified secrets with datasources

* Fix minor issues and tests for kvstore

* Create test service helper for secret store

* Remove encryption tests from datasources

* Move secret operations after datasources

* Fix datasource proxy tests

* Fix legacy data tests

* Add Name to all delete data source commands

* Implement decryption cache on sql secret store

* Fix minor issue with cache and tests

* Use secret type on secret store datasource operations

* Add comments to make create and update clear

* Rename itemFound variable to isFound

* Improve secret deletion and cache management

* Add base64 encoding to sql secret store

* Move secret retrieval to decrypted values function

* Refactor decrypt secure json data functions

* Fix expr tests

* Fix datasource tests

* Fix plugin proxy tests

* Fix query tests

* Fix metrics api tests

* Remove unused fake secrets service from query tests

* Add rename function to secret store

* Add check for error renaming secret

* Remove bus from tests to fix merge conflicts

* Add background secrets migration to datasources

* Get datasource secure json fields from secrets

* Move migration to secret store

* Revert "Move migration to secret store"

This reverts commit 7c3f872072.

* Add secret service to datasource service on tests

* Fix datasource tests

* Remove merge conflict on wire

* Add ctx to data source http transport on prometheus stats collector

* Add ctx to data source http transport on stats collector test

pkg/services/datasources/datasources.go

@@ -33,23 +33,23 @@ type DataSourceService interface {
 	GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error
 
 	// GetHTTPTransport gets a datasource specific HTTP transport.
-	GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error)
+	GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error)
 
 	// DecryptedValues decrypts the encrypted secureJSONData of the provided datasource and
 	// returns the decrypted values.
-	DecryptedValues(ds *models.DataSource) map[string]string
+	DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error)
 
 	// DecryptedValue decrypts the encrypted datasource secureJSONData identified by key
 	// and returns the decryped value.
-	DecryptedValue(ds *models.DataSource, key string) (string, bool)
+	DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error)
 
 	// DecryptedBasicAuthPassword decrypts the encrypted datasource basic authentication
 	// password and returns the decryped value.
-	DecryptedBasicAuthPassword(ds *models.DataSource) string
+	DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error)
 
 	// DecryptedPassword decrypts the encrypted datasource password and returns the
 	// decryped value.
-	DecryptedPassword(ds *models.DataSource) string
+	DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error)
 }
 
 // CacheService interface for retrieving a cached datasource.

pkg/services/datasources/fakes/fake_cache_service.go

@@ -4,13 +4,14 @@ import (
 	"context"
 
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/datasources"
 )
 
 type FakeCacheService struct {
 	DataSources []*models.DataSource
 }
 
-var _ CacheService = &FakeCacheService{}
+var _ datasources.CacheService = &FakeCacheService{}
 
 func (c *FakeCacheService) GetDatasource(ctx context.Context, datasourceID int64, user *models.SignedInUser, skipCache bool) (*models.DataSource, error) {
 	for _, datasource := range c.DataSources {

pkg/services/datasources/fakes/fake_datasource_service.go

@@ -0,0 +1,124 @@
+package datasources
+
+import (
+	"context"
+	"net/http"
+
+	sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/datasources"
+)
+
+type FakeDataSourceService struct {
+	lastId      int64
+	DataSources []*models.DataSource
+}
+
+var _ datasources.DataSourceService = &FakeDataSourceService{}
+
+func (s *FakeDataSourceService) GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error {
+	for _, datasource := range s.DataSources {
+		idMatch := query.Id != 0 && query.Id == datasource.Id
+		uidMatch := query.Uid != "" && query.Uid == datasource.Uid
+		nameMatch := query.Name != "" && query.Name == datasource.Name
+		if idMatch || nameMatch || uidMatch {
+			query.Result = datasource
+
+			return nil
+		}
+	}
+	return models.ErrDataSourceNotFound
+}
+
+func (s *FakeDataSourceService) GetDataSources(ctx context.Context, query *models.GetDataSourcesQuery) error {
+	for _, datasource := range s.DataSources {
+		orgMatch := query.OrgId != 0 && query.OrgId == datasource.OrgId
+		if orgMatch {
+			query.Result = append(query.Result, datasource)
+		}
+	}
+	return nil
+}
+
+func (s *FakeDataSourceService) GetDataSourcesByType(ctx context.Context, query *models.GetDataSourcesByTypeQuery) error {
+	for _, datasource := range s.DataSources {
+		typeMatch := query.Type != "" && query.Type == datasource.Type
+		if typeMatch {
+			query.Result = append(query.Result, datasource)
+		}
+	}
+	return nil
+}
+
+func (s *FakeDataSourceService) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCommand) error {
+	if s.lastId == 0 {
+		s.lastId = int64(len(s.DataSources) - 1)
+	}
+	cmd.Result = &models.DataSource{
+		Id:    s.lastId + 1,
+		Name:  cmd.Name,
+		Type:  cmd.Type,
+		Uid:   cmd.Uid,
+		OrgId: cmd.OrgId,
+	}
+	s.DataSources = append(s.DataSources, cmd.Result)
+	return nil
+}
+
+func (s *FakeDataSourceService) DeleteDataSource(ctx context.Context, cmd *models.DeleteDataSourceCommand) error {
+	for i, datasource := range s.DataSources {
+		idMatch := cmd.ID != 0 && cmd.ID == datasource.Id
+		uidMatch := cmd.UID != "" && cmd.UID == datasource.Uid
+		nameMatch := cmd.Name != "" && cmd.Name == datasource.Name
+		if idMatch || nameMatch || uidMatch {
+			s.DataSources = append(s.DataSources[:i], s.DataSources[i+1:]...)
+			return nil
+		}
+	}
+	return models.ErrDataSourceNotFound
+}
+
+func (s *FakeDataSourceService) UpdateDataSource(ctx context.Context, cmd *models.UpdateDataSourceCommand) error {
+	for _, datasource := range s.DataSources {
+		idMatch := cmd.Id != 0 && cmd.Id == datasource.Id
+		uidMatch := cmd.Uid != "" && cmd.Uid == datasource.Uid
+		nameMatch := cmd.Name != "" && cmd.Name == datasource.Name
+		if idMatch || nameMatch || uidMatch {
+			if cmd.Name != "" {
+				datasource.Name = cmd.Name
+			}
+			return nil
+		}
+	}
+	return models.ErrDataSourceNotFound
+}
+
+func (s *FakeDataSourceService) GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error {
+	return nil
+}
+
+func (s *FakeDataSourceService) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) {
+	rt, err := provider.GetTransport(sdkhttpclient.Options{})
+	if err != nil {
+		return nil, err
+	}
+	return rt, nil
+}
+
+func (s *FakeDataSourceService) DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) {
+	values := make(map[string]string)
+	return values, nil
+}
+
+func (s *FakeDataSourceService) DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error) {
+	return "", false, nil
+}
+
+func (s *FakeDataSourceService) DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error) {
+	return "", nil
+}
+
+func (s *FakeDataSourceService) DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error) {
+	return "", nil
+}

pkg/services/datasources/service/datasource_service.go

@@ -3,6 +3,7 @@ package service
 import (
 	"context"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -23,20 +24,21 @@ import (
 	"github.com/grafana/grafana/pkg/services/datasources"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/secrets"
+	"github.com/grafana/grafana/pkg/services/secrets/kvstore"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
 type Service struct {
 	SQLStore           *sqlstore.SQLStore
+	SecretsStore       kvstore.SecretsKVStore
 	SecretsService     secrets.Service
 	cfg                *setting.Cfg
 	features           featuremgmt.FeatureToggles
 	permissionsService accesscontrol.PermissionsService
 	ac                 accesscontrol.AccessControl
 
-	ptc               proxyTransportCache
-	dsDecryptionCache secureJSONDecryptionCache
+	ptc proxyTransportCache
 }
 
 type proxyTransportCache struct {
@@ -49,29 +51,17 @@ type cachedRoundTripper struct {
 	roundTripper http.RoundTripper
 }
 
-type secureJSONDecryptionCache struct {
-	cache map[int64]cachedDecryptedJSON
-	sync.Mutex
-}
-
-type cachedDecryptedJSON struct {
-	updated time.Time
-	json    map[string]string
-}
-
 func ProvideService(
-	store *sqlstore.SQLStore, secretsService secrets.Service, cfg *setting.Cfg, features featuremgmt.FeatureToggles,
-	ac accesscontrol.AccessControl, permissionsServices accesscontrol.PermissionsServices,
+	store *sqlstore.SQLStore, secretsService secrets.Service, secretsStore kvstore.SecretsKVStore, cfg *setting.Cfg,
+	features featuremgmt.FeatureToggles, ac accesscontrol.AccessControl, permissionsServices accesscontrol.PermissionsServices,
 ) *Service {
 	s := &Service{
 		SQLStore:       store,
+		SecretsStore:   secretsStore,
 		SecretsService: secretsService,
 		ptc: proxyTransportCache{
 			cache: make(map[int64]cachedRoundTripper),
 		},
-		dsDecryptionCache: secureJSONDecryptionCache{
-			cache: make(map[int64]cachedDecryptedJSON),
-		},
 		cfg:                cfg,
 		features:           features,
 		permissionsService: permissionsServices.GetDataSourceService(),
@@ -90,6 +80,8 @@ type DataSourceRetriever interface {
 	GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error
 }
 
+const secretType = "datasource"
+
 // NewNameScopeResolver provides an AttributeScopeResolver able to
 // translate a scope prefixed with "datasources:name:" into an uid based scope.
 func NewNameScopeResolver(db DataSourceRetriever) (string, accesscontrol.AttributeScopeResolveFunc) {
@@ -155,12 +147,17 @@ func (s *Service) GetDataSourcesByType(ctx context.Context, query *models.GetDat
 
 func (s *Service) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCommand) error {
 	var err error
-	cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope())
+	if err := s.SQLStore.AddDataSource(ctx, cmd); err != nil {
+		return err
+	}
+
+	secret, err := json.Marshal(cmd.SecureJsonData)
 	if err != nil {
 		return err
 	}
 
-	if err := s.SQLStore.AddDataSource(ctx, cmd); err != nil {
+	err = s.SecretsStore.Set(ctx, cmd.OrgId, cmd.Name, secretType, string(secret))
+	if err != nil {
 		return err
 	}
 
@@ -186,25 +183,50 @@ func (s *Service) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCo
 }
 
 func (s *Service) DeleteDataSource(ctx context.Context, cmd *models.DeleteDataSourceCommand) error {
-	return s.SQLStore.DeleteDataSource(ctx, cmd)
+	err := s.SQLStore.DeleteDataSource(ctx, cmd)
+	if err != nil {
+		return err
+	}
+	return s.SecretsStore.Del(ctx, cmd.OrgID, cmd.Name, secretType)
 }
 
 func (s *Service) UpdateDataSource(ctx context.Context, cmd *models.UpdateDataSourceCommand) error {
 	var err error
-	cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope())
+	secret, err := json.Marshal(cmd.SecureJsonData)
 	if err != nil {
 		return err
 	}
 
-	return s.SQLStore.UpdateDataSource(ctx, cmd)
+	query := &models.GetDataSourceQuery{
+		Id:    cmd.Id,
+		OrgId: cmd.OrgId,
+	}
+	err = s.SQLStore.GetDataSource(ctx, query)
+	if err != nil {
+		return err
+	}
+
+	err = s.SQLStore.UpdateDataSource(ctx, cmd)
+	if err != nil {
+		return err
+	}
+
+	if query.Result.Name != cmd.Name {
+		err = s.SecretsStore.Rename(ctx, cmd.OrgId, query.Result.Name, secretType, cmd.Name)
+		if err != nil {
+			return err
+		}
+	}
+
+	return s.SecretsStore.Set(ctx, cmd.OrgId, cmd.Name, secretType, string(secret))
 }
 
 func (s *Service) GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error {
 	return s.SQLStore.GetDefaultDataSource(ctx, query)
 }
 
-func (s *Service) GetHTTPClient(ds *models.DataSource, provider httpclient.Provider) (*http.Client, error) {
-	transport, err := s.GetHTTPTransport(ds, provider)
+func (s *Service) GetHTTPClient(ctx context.Context, ds *models.DataSource, provider httpclient.Provider) (*http.Client, error) {
+	transport, err := s.GetHTTPTransport(ctx, ds, provider)
 	if err != nil {
 		return nil, err
 	}
@@ -215,7 +237,7 @@ func (s *Service) GetHTTPClient(ds *models.DataSource, provider httpclient.Provi
 	}, nil
 }
 
-func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider,
+func (s *Service) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider,
 	customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) {
 	s.ptc.Lock()
 	defer s.ptc.Unlock()
@@ -224,7 +246,7 @@ func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Pr
 		return t.roundTripper, nil
 	}
 
-	opts, err := s.httpClientOptions(ds)
+	opts, err := s.httpClientOptions(ctx, ds)
 	if err != nil {
 		return nil, err
 	}
@@ -244,58 +266,84 @@ func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Pr
 	return rt, nil
 }
 
-func (s *Service) GetTLSConfig(ds *models.DataSource, httpClientProvider httpclient.Provider) (*tls.Config, error) {
-	opts, err := s.httpClientOptions(ds)
+func (s *Service) GetTLSConfig(ctx context.Context, ds *models.DataSource, httpClientProvider httpclient.Provider) (*tls.Config, error) {
+	opts, err := s.httpClientOptions(ctx, ds)
 	if err != nil {
 		return nil, err
 	}
 	return httpClientProvider.GetTLSConfig(*opts)
 }
 
-func (s *Service) DecryptedValues(ds *models.DataSource) map[string]string {
-	s.dsDecryptionCache.Lock()
-	defer s.dsDecryptionCache.Unlock()
-
-	if item, present := s.dsDecryptionCache.cache[ds.Id]; present && ds.Updated.Equal(item.updated) {
-		return item.json
-	}
-
-	json, err := s.SecretsService.DecryptJsonData(context.Background(), ds.SecureJsonData)
+func (s *Service) DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) {
+	decryptedValues := make(map[string]string)
+	secret, exist, err := s.SecretsStore.Get(ctx, ds.OrgId, ds.Name, secretType)
 	if err != nil {
-		return map[string]string{}
+		return nil, err
 	}
 
-	s.dsDecryptionCache.cache[ds.Id] = cachedDecryptedJSON{
-		updated: ds.Updated,
-		json:    json,
+	if exist {
+		err := json.Unmarshal([]byte(secret), &decryptedValues)
+		if err != nil {
+			return nil, err
+		}
+	} else if len(ds.SecureJsonData) > 0 {
+		decryptedValues, err = s.MigrateSecrets(ctx, ds)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	return json
+	return decryptedValues, nil
 }
 
-func (s *Service) DecryptedValue(ds *models.DataSource, key string) (string, bool) {
-	value, exists := s.DecryptedValues(ds)[key]
-	return value, exists
-}
-
-func (s *Service) DecryptedBasicAuthPassword(ds *models.DataSource) string {
-	if value, ok := s.DecryptedValue(ds, "basicAuthPassword"); ok {
-		return value
+func (s *Service) MigrateSecrets(ctx context.Context, ds *models.DataSource) (map[string]string, error) {
+	secureJsonData, err := s.SecretsService.DecryptJsonData(ctx, ds.SecureJsonData)
+	if err != nil {
+		return nil, err
 	}
 
-	return ds.BasicAuthPassword
-}
-
-func (s *Service) DecryptedPassword(ds *models.DataSource) string {
-	if value, ok := s.DecryptedValue(ds, "password"); ok {
-		return value
+	jsonData, err := json.Marshal(secureJsonData)
+	if err != nil {
+		return nil, err
 	}
 
-	return ds.Password
+	err = s.SecretsStore.Set(ctx, ds.OrgId, ds.Name, secretType, string(jsonData))
+	return secureJsonData, err
 }
 
-func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Options, error) {
-	tlsOptions := s.dsTLSOptions(ds)
+func (s *Service) DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error) {
+	values, err := s.DecryptedValues(ctx, ds)
+	if err != nil {
+		return "", false, err
+	}
+	value, exists := values[key]
+	return value, exists, nil
+}
+
+func (s *Service) DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error) {
+	value, ok, err := s.DecryptedValue(ctx, ds, "basicAuthPassword")
+	if ok {
+		return value, nil
+	}
+
+	return ds.BasicAuthPassword, err
+}
+
+func (s *Service) DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error) {
+	value, ok, err := s.DecryptedValue(ctx, ds, "password")
+	if ok {
+		return value, nil
+	}
+
+	return ds.Password, err
+}
+
+func (s *Service) httpClientOptions(ctx context.Context, ds *models.DataSource) (*sdkhttpclient.Options, error) {
+	tlsOptions, err := s.dsTLSOptions(ctx, ds)
+	if err != nil {
+		return nil, err
+	}
+
 	timeouts := &sdkhttpclient.TimeoutOptions{
 		Timeout:               s.getTimeout(ds),
 		DialTimeout:           sdkhttpclient.DefaultTimeoutOptions.DialTimeout,
@@ -307,9 +355,15 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio
 		MaxIdleConnsPerHost:   sdkhttpclient.DefaultTimeoutOptions.MaxIdleConnsPerHost,
 		IdleConnTimeout:       sdkhttpclient.DefaultTimeoutOptions.IdleConnTimeout,
 	}
+
+	decryptedValues, err := s.DecryptedValues(ctx, ds)
+	if err != nil {
+		return nil, err
+	}
+
 	opts := &sdkhttpclient.Options{
 		Timeouts: timeouts,
-		Headers:  s.getCustomHeaders(ds.JsonData, s.DecryptedValues(ds)),
+		Headers:  s.getCustomHeaders(ds.JsonData, decryptedValues),
 		Labels: map[string]string{
 			"datasource_name": ds.Name,
 			"datasource_uid":  ds.Uid,
@@ -320,22 +374,30 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio
 	if ds.JsonData != nil {
 		opts.CustomOptions = ds.JsonData.MustMap()
 	}
-
 	if ds.BasicAuth {
+		password, err := s.DecryptedBasicAuthPassword(ctx, ds)
+		if err != nil {
+			return opts, err
+		}
+
 		opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{
 			User:     ds.BasicAuthUser,
-			Password: s.DecryptedBasicAuthPassword(ds),
+			Password: password,
 		}
 	} else if ds.User != "" {
+		password, err := s.DecryptedPassword(ctx, ds)
+		if err != nil {
+			return opts, err
+		}
+
 		opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{
 			User:     ds.User,
-			Password: s.DecryptedPassword(ds),
+			Password: password,
 		}
 	}
 
-	// Azure authentication
 	if ds.JsonData != nil && s.features.IsEnabled(featuremgmt.FlagHttpclientproviderAzureAuth) {
-		credentials, err := azcredentials.FromDatasourceData(ds.JsonData.MustMap(), s.DecryptedValues(ds))
+		credentials, err := azcredentials.FromDatasourceData(ds.JsonData.MustMap(), decryptedValues)
 		if err != nil {
 			err = fmt.Errorf("invalid Azure credentials: %s", err)
 			return nil, err
@@ -371,19 +433,27 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio
 			Profile:       ds.JsonData.Get("sigV4Profile").MustString(),
 		}
 
-		if val, exists := s.DecryptedValue(ds, "sigV4AccessKey"); exists {
-			opts.SigV4.AccessKey = val
+		if val, exists, err := s.DecryptedValue(ctx, ds, "sigV4AccessKey"); err == nil {
+			if exists {
+				opts.SigV4.AccessKey = val
+			}
+		} else {
+			return opts, err
 		}
 
-		if val, exists := s.DecryptedValue(ds, "sigV4SecretKey"); exists {
-			opts.SigV4.SecretKey = val
+		if val, exists, err := s.DecryptedValue(ctx, ds, "sigV4SecretKey"); err == nil {
+			if exists {
+				opts.SigV4.SecretKey = val
+			}
+		} else {
+			return opts, err
 		}
 	}
 
 	return opts, nil
 }
 
-func (s *Service) dsTLSOptions(ds *models.DataSource) sdkhttpclient.TLSOptions {
+func (s *Service) dsTLSOptions(ctx context.Context, ds *models.DataSource) (sdkhttpclient.TLSOptions, error) {
 	var tlsSkipVerify, tlsClientAuth, tlsAuthWithCACert bool
 	var serverName string
 
@@ -401,22 +471,35 @@ func (s *Service) dsTLSOptions(ds *models.DataSource) sdkhttpclient.TLSOptions {
 
 	if tlsClientAuth || tlsAuthWithCACert {
 		if tlsAuthWithCACert {
-			if val, exists := s.DecryptedValue(ds, "tlsCACert"); exists && len(val) > 0 {
-				opts.CACertificate = val
+			if val, exists, err := s.DecryptedValue(ctx, ds, "tlsCACert"); err == nil {
+				if exists && len(val) > 0 {
+					opts.CACertificate = val
+				}
+			} else {
+				return opts, err
 			}
 		}
 
 		if tlsClientAuth {
-			if val, exists := s.DecryptedValue(ds, "tlsClientCert"); exists && len(val) > 0 {
-				opts.ClientCertificate = val
+			if val, exists, err := s.DecryptedValue(ctx, ds, "tlsClientCert"); err == nil {
+				fmt.Print("\n\n\n\n", val, exists, err, "\n\n\n\n")
+				if exists && len(val) > 0 {
+					opts.ClientCertificate = val
+				}
+			} else {
+				return opts, err
 			}
-			if val, exists := s.DecryptedValue(ds, "tlsClientKey"); exists && len(val) > 0 {
-				opts.ClientKey = val
+			if val, exists, err := s.DecryptedValue(ctx, ds, "tlsClientKey"); err == nil {
+				if exists && len(val) > 0 {
+					opts.ClientKey = val
+				}
+			} else {
+				return opts, err
 			}
 		}
 	}
 
-	return opts
+	return opts, nil
 }
 
 func (s *Service) getTimeout(ds *models.DataSource) time.Duration {

pkg/services/datasources/service/datasource_service_test.go

@@ -2,6 +2,7 @@ package service
 
 import (
 	"context"
+	encJson "encoding/json"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -10,6 +11,7 @@ import (
 
 	"github.com/grafana/grafana-azure-sdk-go/azsettings"
 	sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/httpclient"
@@ -17,59 +19,13 @@ import (
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
 	acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
-	"github.com/grafana/grafana/pkg/services/secrets"
-	"github.com/grafana/grafana/pkg/services/secrets/database"
 	"github.com/grafana/grafana/pkg/services/secrets/fakes"
-	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
-	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/services/secrets/kvstore"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestService(t *testing.T) {
-	cfg := &setting.Cfg{}
-	sqlStore := sqlstore.InitTestDB(t)
-
-	origSecret := setting.SecretKey
-	setting.SecretKey = "datasources_service_test"
-	t.Cleanup(func() {
-		setting.SecretKey = origSecret
-	})
-
-	secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(sqlStore))
-	s := ProvideService(sqlStore, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New().WithDisabled(), acmock.NewPermissionsServicesMock())
-
-	var ds *models.DataSource
-
-	t.Run("create datasource should encrypt the secure json data", func(t *testing.T) {
-		ctx := context.Background()
-
-		sjd := map[string]string{"password": "12345"}
-		cmd := models.AddDataSourceCommand{SecureJsonData: sjd}
-
-		err := s.AddDataSource(ctx, &cmd)
-		require.NoError(t, err)
-
-		ds = cmd.Result
-		decrypted, err := s.SecretsService.DecryptJsonData(ctx, ds.SecureJsonData)
-		require.NoError(t, err)
-		require.Equal(t, sjd, decrypted)
-	})
-
-	t.Run("update datasource should encrypt the secure json data", func(t *testing.T) {
-		ctx := context.Background()
-		sjd := map[string]string{"password": "678910"}
-		cmd := models.UpdateDataSourceCommand{Id: ds.Id, OrgId: ds.OrgId, SecureJsonData: sjd}
-		err := s.UpdateDataSource(ctx, &cmd)
-		require.NoError(t, err)
-
-		decrypted, err := s.SecretsService.DecryptJsonData(ctx, cmd.Result.SecureJsonData)
-		require.NoError(t, err)
-		require.Equal(t, sjd, decrypted)
-	})
-}
-
 type dataSourceMockRetriever struct {
 	res []*models.DataSource
 }
@@ -237,15 +193,16 @@ func TestService_GetHttpTransport(t *testing.T) {
 			Type: "Kubernetes",
 		}
 
+		secretsStore := kvstore.SetupTestService(t)
 		secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-		dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
+		dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
 
-		rt1, err := dsService.GetHTTPTransport(&ds, provider)
+		rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
 		require.NoError(t, err)
 		require.NotNil(t, rt1)
 		tr1 := configuredTransport
 
-		rt2, err := dsService.GetHTTPTransport(&ds, provider)
+		rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
 		require.NoError(t, err)
 		require.NotNil(t, rt2)
 		tr2 := configuredTransport
@@ -270,21 +227,19 @@ func TestService_GetHttpTransport(t *testing.T) {
 		json := simplejson.New()
 		json.Set("tlsAuthWithCACert", true)
 
+		secretsStore := kvstore.SetupTestService(t)
 		secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-		dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
-
-		tlsCaCert, err := secretsService.Encrypt(context.Background(), []byte(caCert), secrets.WithoutScope())
-		require.NoError(t, err)
+		dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
 
 		ds := models.DataSource{
 			Id:             1,
 			Url:            "http://k8s:8001",
 			Type:           "Kubernetes",
-			SecureJsonData: map[string][]byte{"tlsCACert": tlsCaCert},
+			SecureJsonData: map[string][]byte{"tlsCACert": []byte(caCert)},
 			Updated:        time.Now().Add(-2 * time.Minute),
 		}
 
-		rt1, err := dsService.GetHTTPTransport(&ds, provider)
+		rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
 		require.NotNil(t, rt1)
 		require.NoError(t, err)
 
@@ -298,7 +253,7 @@ func TestService_GetHttpTransport(t *testing.T) {
 		ds.SecureJsonData = map[string][]byte{}
 		ds.Updated = time.Now()
 
-		rt2, err := dsService.GetHTTPTransport(&ds, provider)
+		rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
 		require.NoError(t, err)
 		require.NotNil(t, rt2)
 		tr2 := configuredTransport
@@ -320,27 +275,29 @@ func TestService_GetHttpTransport(t *testing.T) {
 		json := simplejson.New()
 		json.Set("tlsAuth", true)
 
+		secretsStore := kvstore.SetupTestService(t)
 		secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-		dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
-
-		tlsClientCert, err := secretsService.Encrypt(context.Background(), []byte(clientCert), secrets.WithoutScope())
-		require.NoError(t, err)
-
-		tlsClientKey, err := secretsService.Encrypt(context.Background(), []byte(clientKey), secrets.WithoutScope())
-		require.NoError(t, err)
+		dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
 
 		ds := models.DataSource{
 			Id:       1,
+			OrgId:    1,
+			Name:     "kubernetes",
 			Url:      "http://k8s:8001",
 			Type:     "Kubernetes",
 			JsonData: json,
-			SecureJsonData: map[string][]byte{
-				"tlsClientCert": tlsClientCert,
-				"tlsClientKey":  tlsClientKey,
-			},
 		}
 
-		rt, err := dsService.GetHTTPTransport(&ds, provider)
+		secureJsonData, err := encJson.Marshal(map[string]string{
+			"tlsClientCert": clientCert,
+			"tlsClientKey":  clientKey,
+		})
+		require.NoError(t, err)
+
+		err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData))
+		require.NoError(t, err)
+
+		rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
 		require.NoError(t, err)
 		require.NotNil(t, rt)
 		tr := configuredTransport
@@ -363,23 +320,28 @@ func TestService_GetHttpTransport(t *testing.T) {
 		json.Set("tlsAuthWithCACert", true)
 		json.Set("serverName", "server-name")
 
+		secretsStore := kvstore.SetupTestService(t)
 		secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-		dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
-
-		tlsCaCert, err := secretsService.Encrypt(context.Background(), []byte(caCert), secrets.WithoutScope())
-		require.NoError(t, err)
+		dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
 
 		ds := models.DataSource{
 			Id:       1,
+			OrgId:    1,
+			Name:     "kubernetes",
 			Url:      "http://k8s:8001",
 			Type:     "Kubernetes",
 			JsonData: json,
-			SecureJsonData: map[string][]byte{
-				"tlsCACert": tlsCaCert,
-			},
 		}
 
-		rt, err := dsService.GetHTTPTransport(&ds, provider)
+		secureJsonData, err := encJson.Marshal(map[string]string{
+			"tlsCACert": caCert,
+		})
+		require.NoError(t, err)
+
+		err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData))
+		require.NoError(t, err)
+
+		rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
 		require.NoError(t, err)
 		require.NotNil(t, rt)
 		tr := configuredTransport
@@ -400,8 +362,9 @@ func TestService_GetHttpTransport(t *testing.T) {
 		json := simplejson.New()
 		json.Set("tlsSkipVerify", true)
 
+		secretsStore := kvstore.SetupTestService(t)
 		secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-		dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
+		dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
 
 		ds := models.DataSource{
 			Id:       1,
@@ -410,12 +373,12 @@ func TestService_GetHttpTransport(t *testing.T) {
 			JsonData: json,
 		}
 
-		rt1, err := dsService.GetHTTPTransport(&ds, provider)
+		rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
 		require.NoError(t, err)
 		require.NotNil(t, rt1)
 		tr1 := configuredTransport
 
-		rt2, err := dsService.GetHTTPTransport(&ds, provider)
+		rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
 		require.NoError(t, err)
 		require.NotNil(t, rt2)
 		tr2 := configuredTransport
@@ -431,20 +394,27 @@ func TestService_GetHttpTransport(t *testing.T) {
 			"httpHeaderName1": "Authorization",
 		})
 
+		secretsStore := kvstore.SetupTestService(t)
 		secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-		dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
-
-		encryptedData, err := secretsService.Encrypt(context.Background(), []byte(`Bearer xf5yhfkpsnmgo`), secrets.WithoutScope())
-		require.NoError(t, err)
+		dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
 
 		ds := models.DataSource{
-			Id:             1,
-			Url:            "http://k8s:8001",
-			Type:           "Kubernetes",
-			JsonData:       json,
-			SecureJsonData: map[string][]byte{"httpHeaderValue1": encryptedData},
+			Id:       1,
+			OrgId:    1,
+			Name:     "kubernetes",
+			Url:      "http://k8s:8001",
+			Type:     "Kubernetes",
+			JsonData: json,
 		}
 
+		secureJsonData, err := encJson.Marshal(map[string]string{
+			"httpHeaderValue1": "Bearer xf5yhfkpsnmgo",
+		})
+		require.NoError(t, err)
+
+		err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData))
+		require.NoError(t, err)
+
 		headers := dsService.getCustomHeaders(json, map[string]string{"httpHeaderValue1": "Bearer xf5yhfkpsnmgo"})
 		require.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"])
 
@@ -465,7 +435,7 @@ func TestService_GetHttpTransport(t *testing.T) {
 
 		// 2. Get HTTP transport from datasource which uses the test server as backend
 		ds.Url = backend.URL
-		rt, err := dsService.GetHTTPTransport(&ds, provider)
+		rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
 		require.NoError(t, err)
 		require.NotNil(t, rt)
 
@@ -490,8 +460,9 @@ func TestService_GetHttpTransport(t *testing.T) {
 			"timeout": 19,
 		})
 
+		secretsStore := kvstore.SetupTestService(t)
 		secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-		dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
+		dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
 
 		ds := models.DataSource{
 			Id:       1,
@@ -500,7 +471,7 @@ func TestService_GetHttpTransport(t *testing.T) {
 			JsonData: json,
 		}
 
-		client, err := dsService.GetHTTPClient(&ds, provider)
+		client, err := dsService.GetHTTPClient(context.Background(), &ds, provider)
 		require.NoError(t, err)
 		require.NotNil(t, client)
 		require.Equal(t, 19*time.Second, client.Timeout)
@@ -523,15 +494,16 @@ func TestService_GetHttpTransport(t *testing.T) {
 		json, err := simplejson.NewJson([]byte(`{ "sigV4Auth": true }`))
 		require.NoError(t, err)
 
+		secretsStore := kvstore.SetupTestService(t)
 		secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-		dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
+		dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
 
 		ds := models.DataSource{
 			Type:     models.DS_ES,
 			JsonData: json,
 		}
 
-		_, err = dsService.GetHTTPTransport(&ds, provider)
+		_, err = dsService.GetHTTPTransport(context.Background(), &ds, provider)
 		require.NoError(t, err)
 		require.NotNil(t, configuredOpts)
 		require.NotNil(t, configuredOpts.SigV4)
@@ -558,8 +530,9 @@ func TestService_getTimeout(t *testing.T) {
 		{jsonData: simplejson.NewFromAny(map[string]interface{}{"timeout": "2"}), expectedTimeout: 2 * time.Second},
 	}
 
+	secretsStore := kvstore.SetupTestService(t)
 	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-	dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
+	dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
 
 	for _, tc := range testCases {
 		ds := &models.DataSource{
@@ -569,86 +542,6 @@ func TestService_getTimeout(t *testing.T) {
 	}
 }
 
-func TestService_DecryptedValue(t *testing.T) {
-	cfg := &setting.Cfg{}
-
-	t.Run("When datasource hasn't been updated, encrypted JSON should be fetched from cache", func(t *testing.T) {
-		secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-		dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
-
-		encryptedJsonData, err := secretsService.EncryptJsonData(
-			context.Background(),
-			map[string]string{
-				"password": "password",
-			}, secrets.WithoutScope())
-		require.NoError(t, err)
-
-		ds := models.DataSource{
-			Id:             1,
-			Type:           models.DS_INFLUXDB_08,
-			JsonData:       simplejson.New(),
-			User:           "user",
-			SecureJsonData: encryptedJsonData,
-		}
-
-		// Populate cache
-		password, ok := dsService.DecryptedValue(&ds, "password")
-		require.True(t, ok)
-		require.Equal(t, "password", password)
-
-		encryptedJsonData, err = secretsService.EncryptJsonData(
-			context.Background(),
-			map[string]string{
-				"password": "",
-			}, secrets.WithoutScope())
-		require.NoError(t, err)
-
-		ds.SecureJsonData = encryptedJsonData
-
-		password, ok = dsService.DecryptedValue(&ds, "password")
-		require.True(t, ok)
-		require.Equal(t, "password", password)
-	})
-
-	t.Run("When datasource is updated, encrypted JSON should not be fetched from cache", func(t *testing.T) {
-		secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-
-		encryptedJsonData, err := secretsService.EncryptJsonData(
-			context.Background(),
-			map[string]string{
-				"password": "password",
-			}, secrets.WithoutScope())
-		require.NoError(t, err)
-
-		ds := models.DataSource{
-			Id:             1,
-			Type:           models.DS_INFLUXDB_08,
-			JsonData:       simplejson.New(),
-			User:           "user",
-			SecureJsonData: encryptedJsonData,
-		}
-
-		dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
-
-		// Populate cache
-		password, ok := dsService.DecryptedValue(&ds, "password")
-		require.True(t, ok)
-		require.Equal(t, "password", password)
-
-		ds.SecureJsonData, err = secretsService.EncryptJsonData(
-			context.Background(),
-			map[string]string{
-				"password": "",
-			}, secrets.WithoutScope())
-		ds.Updated = time.Now()
-		require.NoError(t, err)
-
-		password, ok = dsService.DecryptedValue(&ds, "password")
-		require.True(t, ok)
-		require.Empty(t, password)
-	})
-}
-
 func TestService_HTTPClientOptions(t *testing.T) {
 	cfg := &setting.Cfg{
 		Azure: &azsettings.AzureSettings{},
@@ -678,10 +571,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
 					"azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5",
 				})
 
+				secretsStore := kvstore.SetupTestService(t)
 				secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-				dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
+				dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
 
-				opts, err := dsService.httpClientOptions(&ds)
+				opts, err := dsService.httpClientOptions(context.Background(), &ds)
 				require.NoError(t, err)
 
 				require.NotNil(t, opts.Middlewares)
@@ -695,10 +589,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
 					"httpMethod": "POST",
 				})
 
+				secretsStore := kvstore.SetupTestService(t)
 				secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-				dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
+				dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
 
-				opts, err := dsService.httpClientOptions(&ds)
+				opts, err := dsService.httpClientOptions(context.Background(), &ds)
 				require.NoError(t, err)
 
 				if opts.Middlewares != nil {
@@ -714,10 +609,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
 					"azureCredentials": "invalid",
 				})
 
+				secretsStore := kvstore.SetupTestService(t)
 				secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-				dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
+				dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
 
-				_, err := dsService.httpClientOptions(&ds)
+				_, err := dsService.httpClientOptions(context.Background(), &ds)
 				assert.Error(t, err)
 			})
 
@@ -732,10 +628,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
 					"azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5",
 				})
 
+				secretsStore := kvstore.SetupTestService(t)
 				secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-				dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
+				dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
 
-				opts, err := dsService.httpClientOptions(&ds)
+				opts, err := dsService.httpClientOptions(context.Background(), &ds)
 				require.NoError(t, err)
 
 				require.NotNil(t, opts.Middlewares)
@@ -750,10 +647,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
 					"azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5",
 				})
 
+				secretsStore := kvstore.SetupTestService(t)
 				secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-				dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
+				dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
 
-				opts, err := dsService.httpClientOptions(&ds)
+				opts, err := dsService.httpClientOptions(context.Background(), &ds)
 				require.NoError(t, err)
 
 				if opts.Middlewares != nil {
@@ -772,10 +670,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
 					"azureEndpointResourceId": "invalid",
 				})
 
+				secretsStore := kvstore.SetupTestService(t)
 				secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-				dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
+				dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
 
-				_, err := dsService.httpClientOptions(&ds)
+				_, err := dsService.httpClientOptions(context.Background(), &ds)
 				assert.Error(t, err)
 			})
 		})
@@ -792,10 +691,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
 					"azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5",
 				})
 
+				secretsStore := kvstore.SetupTestService(t)
 				secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
-				dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
+				dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
 
-				opts, err := dsService.httpClientOptions(&ds)
+				opts, err := dsService.httpClientOptions(context.Background(), &ds)
 				require.NoError(t, err)
 
 				if opts.Middlewares != nil {