Secrets: Implement basic unified secret store service (#45804)

* wip: Implement kvstore for secrets

* wip: Refactor kvstore for secrets

* wip: Add format key function to secrets kvstore sql

* wip: Add migration for secrets kvstore

* Remove unused Key field from secrets kvstore

* Remove secret values from debug logs

* Integrate unified secrets with datasources

* Fix minor issues and tests for kvstore

* Create test service helper for secret store

* Remove encryption tests from datasources

* Move secret operations after datasources

* Fix datasource proxy tests

* Fix legacy data tests

* Add Name to all delete data source commands

* Implement decryption cache on sql secret store

* Fix minor issue with cache and tests

* Use secret type on secret store datasource operations

* Add comments to make create and update clear

* Rename itemFound variable to isFound

* Improve secret deletion and cache management

* Add base64 encoding to sql secret store

* Move secret retrieval to decrypted values function

* Refactor decrypt secure json data functions

* Fix expr tests

* Fix datasource tests

* Fix plugin proxy tests

* Fix query tests

* Fix metrics api tests

* Remove unused fake secrets service from query tests

* Add rename function to secret store

* Add check for error renaming secret

* Remove bus from tests to fix merge conflicts

* Add background secrets migration to datasources

* Get datasource secure json fields from secrets

* Move migration to secret store

* Revert "Move migration to secret store"

This reverts commit 7c3f872072.

* Add secret service to datasource service on tests

* Fix datasource tests

* Remove merge conflict on wire

* Add ctx to data source http transport on prometheus stats collector

* Add ctx to data source http transport on stats collector test

pkg/services/query/query.go

@@ -15,7 +15,6 @@ import (
 	"github.com/grafana/grafana/pkg/plugins/adapters"
 	"github.com/grafana/grafana/pkg/services/datasources"
 	"github.com/grafana/grafana/pkg/services/oauthtoken"
-	"github.com/grafana/grafana/pkg/services/secrets"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb/grafanads"
 	"github.com/grafana/grafana/pkg/tsdb/legacydata"
@@ -33,7 +32,7 @@ func ProvideService(
 	dataSourceCache datasources.CacheService,
 	expressionService *expr.Service,
 	pluginRequestValidator models.PluginRequestValidator,
-	SecretsService secrets.Service,
+	dataSourceService datasources.DataSourceService,
 	pluginClient plugins.Client,
 	oAuthTokenService oauthtoken.OAuthTokenService,
 ) *Service {
@@ -42,7 +41,7 @@ func ProvideService(
 		dataSourceCache:        dataSourceCache,
 		expressionService:      expressionService,
 		pluginRequestValidator: pluginRequestValidator,
-		secretsService:         SecretsService,
+		dataSourceService:      dataSourceService,
 		pluginClient:           pluginClient,
 		oAuthTokenService:      oAuthTokenService,
 		log:                    log.New("query_data"),
@@ -56,7 +55,7 @@ type Service struct {
 	dataSourceCache        datasources.CacheService
 	expressionService      *expr.Service
 	pluginRequestValidator models.PluginRequestValidator
-	secretsService         secrets.Service
+	dataSourceService      datasources.DataSourceService
 	pluginClient           plugins.Client
 	oAuthTokenService      oauthtoken.OAuthTokenService
 	log                    log.Logger
@@ -291,9 +290,9 @@ func (s *Service) getDataSourceFromQuery(ctx context.Context, user *models.Signe
 	return nil, NewErrBadQuery("missing data source ID/UID")
 }
 
-func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(map[string][]byte) map[string]string {
-	return func(m map[string][]byte) map[string]string {
-		decryptedJsonData, err := s.secretsService.DecryptJsonData(ctx, m)
+func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string {
+	return func(ds *models.DataSource) map[string]string {
+		decryptedJsonData, err := s.dataSourceService.DecryptedValues(ctx, ds)
 		if err != nil {
 			s.log.Error("Failed to decrypt secure json data", "error", err)
 		}

pkg/services/query/query_test.go

@@ -2,6 +2,7 @@ package query_test
 
 import (
 	"context"
+	"encoding/json"
 	"net/http"
 	"testing"
 
@@ -12,18 +13,29 @@ import (
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
+	acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
+	datasources "github.com/grafana/grafana/pkg/services/datasources/service"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/query"
-	"github.com/grafana/grafana/pkg/services/secrets"
+	"github.com/grafana/grafana/pkg/services/secrets/fakes"
+	"github.com/grafana/grafana/pkg/services/secrets/kvstore"
+	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
+
 	"github.com/stretchr/testify/require"
 )
 
 func TestQueryData(t *testing.T) {
 	t.Run("it attaches custom headers to the request", func(t *testing.T) {
-		tc := setup()
+		tc := setup(t)
 		tc.dataSourceCache.ds.JsonData = simplejson.NewFromAny(map[string]interface{}{"httpHeaderName1": "foo", "httpHeaderName2": "bar"})
-		tc.secretService.decryptedJson = map[string]string{"httpHeaderValue1": "test-header", "httpHeaderValue2": "test-header2"}
 
-		_, err := tc.queryService.QueryData(context.Background(), nil, true, metricRequest(), false)
+		secureJsonData, err := json.Marshal(map[string]string{"httpHeaderValue1": "test-header", "httpHeaderValue2": "test-header2"})
+		require.NoError(t, err)
+
+		err = tc.secretStore.Set(context.Background(), tc.dataSourceCache.ds.OrgId, tc.dataSourceCache.ds.Name, "datasource", string(secureJsonData))
+		require.NoError(t, err)
+
+		_, err = tc.queryService.QueryData(context.Background(), nil, true, metricRequest(), false)
 		require.Nil(t, err)
 
 		require.Equal(t, map[string]string{"foo": "test-header", "bar": "test-header2"}, tc.pluginContext.req.Headers)
@@ -36,7 +48,7 @@ func TestQueryData(t *testing.T) {
 		}
 		token = token.WithExtra(map[string]interface{}{"id_token": "id-token"})
 
-		tc := setup()
+		tc := setup(t)
 		tc.oauthTokenService.passThruEnabled = true
 		tc.oauthTokenService.token = token
 
@@ -51,26 +63,29 @@ func TestQueryData(t *testing.T) {
 	})
 }
 
-func setup() *testContext {
+func setup(t *testing.T) *testContext {
 	pc := &fakePluginClient{}
-	sc := &fakeSecretsService{}
 	dc := &fakeDataSourceCache{ds: &models.DataSource{}}
 	tc := &fakeOAuthTokenService{}
 	rv := &fakePluginRequestValidator{}
 
+	ss := kvstore.SetupTestService(t)
+	ssvc := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
+	ds := datasources.ProvideService(nil, ssvc, ss, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
+
 	return &testContext{
 		pluginContext:          pc,
-		secretService:          sc,
+		secretStore:            ss,
 		dataSourceCache:        dc,
 		oauthTokenService:      tc,
 		pluginRequestValidator: rv,
-		queryService:           query.ProvideService(nil, dc, nil, rv, sc, pc, tc),
+		queryService:           query.ProvideService(nil, dc, nil, rv, ds, pc, tc),
 	}
 }
 
 type testContext struct {
 	pluginContext          *fakePluginClient
-	secretService          *fakeSecretsService
+	secretStore            kvstore.SecretsKVStore
 	dataSourceCache        *fakeDataSourceCache
 	oauthTokenService      *fakeOAuthTokenService
 	pluginRequestValidator *fakePluginRequestValidator
@@ -108,16 +123,6 @@ func (ts *fakeOAuthTokenService) IsOAuthPassThruEnabled(*models.DataSource) bool
 	return ts.passThruEnabled
 }
 
-type fakeSecretsService struct {
-	secrets.Service
-
-	decryptedJson map[string]string
-}
-
-func (s *fakeSecretsService) DecryptJsonData(ctx context.Context, sjd map[string][]byte) (map[string]string, error) {
-	return s.decryptedJson, nil
-}
-
 type fakeDataSourceCache struct {
 	ds *models.DataSource
 }