IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Secrets: Implement basic unified secret store service (#45804)

Browse Source 
* wip: Implement kvstore for secrets

* wip: Refactor kvstore for secrets

* wip: Add format key function to secrets kvstore sql

* wip: Add migration for secrets kvstore

* Remove unused Key field from secrets kvstore

* Remove secret values from debug logs

* Integrate unified secrets with datasources

* Fix minor issues and tests for kvstore

* Create test service helper for secret store

* Remove encryption tests from datasources

* Move secret operations after datasources

* Fix datasource proxy tests

* Fix legacy data tests

* Add Name to all delete data source commands

* Implement decryption cache on sql secret store

* Fix minor issue with cache and tests

* Use secret type on secret store datasource operations

* Add comments to make create and update clear

* Rename itemFound variable to isFound

* Improve secret deletion and cache management

* Add base64 encoding to sql secret store

* Move secret retrieval to decrypted values function

* Refactor decrypt secure json data functions

* Fix expr tests

* Fix datasource tests

* Fix plugin proxy tests

* Fix query tests

* Fix metrics api tests

* Remove unused fake secrets service from query tests

* Add rename function to secret store

* Add check for error renaming secret

* Remove bus from tests to fix merge conflicts

* Add background secrets migration to datasources

* Get datasource secure json fields from secrets

* Move migration to secret store

* Revert "Move migration to secret store"

This reverts commit 7c3f872072.

* Add secret service to datasource service on tests

* Fix datasource tests

* Remove merge conflict on wire

* Add ctx to data source http transport on prometheus stats collector

* Add ctx to data source http transport on stats collector test
This commit is contained in:
Guilherme Caulada
2022-04-25 13:57:45 -03:00
committed by GitHub
parent 0ca32f0c61
commit a367ad730c
31 changed files with 1243 additions and 452 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
pkg/api/datasources.go
View File

@@ -97,7 +97,7 @@ func (hs *HTTPServer) GetDataSourceById(c *models.ReqContext) response.Response
return response.Error(404, "Data source not found", err)
}
dto := convertModelToDtos(filtered[0])
dto := hs.convertModelToDtos(c.Req.Context(), filtered[0])
// Add accesscontrol metadata
dto.AccessControl = hs.getAccessControlMetadata(c, c.OrgId, datasources.ScopePrefix, dto.UID)
@@ -128,7 +128,7 @@ func (hs *HTTPServer) DeleteDataSourceById(c *models.ReqContext) response.Respon
return response.Error(403, "Cannot delete read-only data source", nil)
}
cmd := &models.DeleteDataSourceCommand{ID: id, OrgID: c.OrgId}
cmd := &models.DeleteDataSourceCommand{ID: id, OrgID: c.OrgId, Name: ds.Name}
err = hs.DataSourcesService.DeleteDataSource(c.Req.Context(), cmd)
if err != nil {
@@ -156,7 +156,7 @@ func (hs *HTTPServer) GetDataSourceByUID(c *models.ReqContext) response.Response
return response.Error(404, "Data source not found", err)
}
dto := convertModelToDtos(filtered[0])
dto := hs.convertModelToDtos(c.Req.Context(), filtered[0])
// Add accesscontrol metadata
dto.AccessControl = hs.getAccessControlMetadata(c, c.OrgId, datasources.ScopePrefix, dto.UID)
@@ -184,7 +184,7 @@ func (hs *HTTPServer) DeleteDataSourceByUID(c *models.ReqContext) response.Respo
return response.Error(403, "Cannot delete read-only data source", nil)
}
cmd := &models.DeleteDataSourceCommand{UID: uid, OrgID: c.OrgId}
cmd := &models.DeleteDataSourceCommand{UID: uid, OrgID: c.OrgId, Name: ds.Name}
err = hs.DataSourcesService.DeleteDataSource(c.Req.Context(), cmd)
if err != nil {
@@ -265,7 +265,7 @@ func (hs *HTTPServer) AddDataSource(c *models.ReqContext) response.Response {
return response.Error(500, "Failed to add datasource", err)
}
ds := convertModelToDtos(cmd.Result)
ds := hs.convertModelToDtos(c.Req.Context(), cmd.Result)
return response.JSON(http.StatusOK, util.DynMap{
"message": "Datasource added",
"id": cmd.Result.Id,
@@ -327,7 +327,7 @@ func (hs *HTTPServer) UpdateDataSource(c *models.ReqContext) response.Response {
return response.Error(500, "Failed to query datasource", err)
}
datasourceDTO := convertModelToDtos(query.Result)
datasourceDTO := hs.convertModelToDtos(c.Req.Context(), query.Result)
hs.Live.HandleDatasourceUpdate(c.OrgId, datasourceDTO.UID)
@@ -408,7 +408,7 @@ func (hs *HTTPServer) GetDataSourceByName(c *models.ReqContext) response.Respons
return response.Error(404, "Data source not found", err)
}
dto := convertModelToDtos(filtered[0])
dto := hs.convertModelToDtos(c.Req.Context(), filtered[0])
return response.JSON(http.StatusOK, &dto)
}
@@ -457,7 +457,7 @@ func (hs *HTTPServer) CallDatasourceResource(c *models.ReqContext) {
hs.callPluginResource(c, plugin.ID, ds.Uid)
}
func convertModelToDtos(ds *models.DataSource) dtos.DataSource {
func (hs *HTTPServer) convertModelToDtos(ctx context.Context, ds *models.DataSource) dtos.DataSource {
dto := dtos.DataSource{
Id: ds.Id,
UID: ds.Uid,
@@ -480,9 +480,12 @@ func convertModelToDtos(ds *models.DataSource) dtos.DataSource {
ReadOnly: ds.ReadOnly,
}
for k, v := range ds.SecureJsonData {
if len(v) > 0 {
dto.SecureJsonFields[k] = true
secrets, err := hs.DataSourcesService.DecryptedValues(ctx, ds)
if err == nil {
for k, v := range secrets {
if len(v) > 0 {
dto.SecureJsonFields[k] = true
}
}
}
@@ -510,7 +513,7 @@ func (hs *HTTPServer) CheckDatasourceHealth(c *models.ReqContext) response.Respo
return response.Error(http.StatusInternalServerError, "Unable to find datasource plugin", err)
}
dsInstanceSettings, err := adapters.ModelToInstanceSettings(ds, hs.decryptSecureJsonDataFn())
dsInstanceSettings, err := adapters.ModelToInstanceSettings(ds, hs.decryptSecureJsonDataFn(c.Req.Context()))
if err != nil {
return response.Error(http.StatusInternalServerError, "Unable to get datasource model", err)
}
@@ -561,9 +564,9 @@ func (hs *HTTPServer) CheckDatasourceHealth(c *models.ReqContext) response.Respo
return response.JSON(http.StatusOK, payload)
}
func (hs *HTTPServer) decryptSecureJsonDataFn() func(map[string][]byte) map[string]string {
return func(m map[string][]byte) map[string]string {
decryptedJsonData, err := hs.SecretsService.DecryptJsonData(context.Background(), m)
func (hs *HTTPServer) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string {
return func(ds *models.DataSource) map[string]string {
decryptedJsonData, err := hs.DataSourcesService.DecryptedValues(ctx, ds)
if err != nil {
hs.log.Error("Failed to decrypt secure json data", "error", err)
}

5
pkg/api/datasources_test.go
View File

@@ -583,3 +583,8 @@ func (m *dataSourcesServiceMock) UpdateDataSource(ctx context.Context, cmd *mode
cmd.Result = m.expectedDatasource
return m.expectedError
}
func (m *dataSourcesServiceMock) DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) {
decryptedValues := make(map[string]string)
return decryptedValues, m.expectedError
}

21
pkg/api/frontendsettings.go
View File

@@ -248,9 +248,14 @@ func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins Enab
if ds.Access == models.DS_ACCESS_DIRECT {
if ds.BasicAuth {
password, err := hs.DataSourcesService.DecryptedBasicAuthPassword(c.Req.Context(), ds)
if err != nil {
return nil, err
}
dsDTO.BasicAuth = util.GetBasicAuthHeader(
ds.BasicAuthUser,
hs.DataSourcesService.DecryptedBasicAuthPassword(ds),
password,
)
}
if ds.WithCredentials {
@@ -258,14 +263,24 @@ func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins Enab
}
if ds.Type == models.DS_INFLUXDB_08 {
password, err := hs.DataSourcesService.DecryptedPassword(c.Req.Context(), ds)
if err != nil {
return nil, err
}
dsDTO.Username = ds.User
dsDTO.Password = hs.DataSourcesService.DecryptedPassword(ds)
dsDTO.Password = password
dsDTO.URL = url + "/db/" + ds.Database
}
if ds.Type == models.DS_INFLUXDB {
password, err := hs.DataSourcesService.DecryptedPassword(c.Req.Context(), ds)
if err != nil {
return nil, err
}
dsDTO.Username = ds.User
dsDTO.Password = hs.DataSourcesService.DecryptedPassword(ds)
dsDTO.Password = password
dsDTO.URL = url
}
}

10
pkg/api/metrics_test.go
View File

@@ -9,6 +9,7 @@ import (
"strings"
"testing"
acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/sqlstore/mockstore"
@@ -18,8 +19,11 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
datasources "github.com/grafana/grafana/pkg/services/datasources/service"
"github.com/grafana/grafana/pkg/services/query"
"github.com/grafana/grafana/pkg/services/secrets/fakes"
"github.com/grafana/grafana/pkg/services/secrets/kvstore"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/stretchr/testify/assert"
)
@@ -193,13 +197,17 @@ type dashboardFakePluginClient struct {
func TestAPIEndpoint_Metrics_QueryMetricsFromDashboard(t *testing.T) {
sc := setupHTTPServerWithMockDb(t, false, false)
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
ds := datasources.ProvideService(nil, secretsService, secretsStore, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
setInitCtxSignedInViewer(sc.initCtx)
sc.hs.queryDataService = query.ProvideService(
nil,
nil,
nil,
&fakePluginRequestValidator{},
fakes.NewFakeSecretsService(),
ds,
&dashboardFakePluginClient{},
&fakeOAuthTokenService{},
)

47
pkg/api/pluginproxy/ds_proxy.go
View File

@@ -19,7 +19,6 @@ import (
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/oauthtoken"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
"github.com/grafana/grafana/pkg/util/proxyutil"
@@ -43,7 +42,6 @@ type DataSourceProxy struct {
oAuthTokenService oauthtoken.OAuthTokenService
dataSourcesService datasources.DataSourceService
tracer tracing.Tracer
secretsService secrets.Service
}
type httpClient interface {
@@ -54,7 +52,7 @@ type httpClient interface {
func NewDataSourceProxy(ds *models.DataSource, pluginRoutes []*plugins.Route, ctx *models.ReqContext,
proxyPath string, cfg *setting.Cfg, clientProvider httpclient.Provider,
oAuthTokenService oauthtoken.OAuthTokenService, dsService datasources.DataSourceService,
tracer tracing.Tracer, secretsService secrets.Service) (*DataSourceProxy, error) {
tracer tracing.Tracer) (*DataSourceProxy, error) {
targetURL, err := datasource.ValidateURL(ds.Type, ds.Url)
if err != nil {
return nil, err
@@ -71,7 +69,6 @@ func NewDataSourceProxy(ds *models.DataSource, pluginRoutes []*plugins.Route, ct
oAuthTokenService: oAuthTokenService,
dataSourcesService: dsService,
tracer: tracer,
secretsService: secretsService,
}, nil
}
@@ -97,7 +94,7 @@ func (proxy *DataSourceProxy) HandleRequest() {
"referer", proxy.ctx.Req.Referer(),
)
transport, err := proxy.dataSourcesService.GetHTTPTransport(proxy.ds, proxy.clientProvider)
transport, err := proxy.dataSourcesService.GetHTTPTransport(proxy.ctx.Req.Context(), proxy.ds, proxy.clientProvider)
if err != nil {
proxy.ctx.JsonApiErr(400, "Unable to load TLS certificate", err)
return
@@ -169,17 +166,28 @@ func (proxy *DataSourceProxy) director(req *http.Request) {
switch proxy.ds.Type {
case models.DS_INFLUXDB_08:
password, err := proxy.dataSourcesService.DecryptedPassword(req.Context(), proxy.ds)
if err != nil {
logger.Error("Error interpolating proxy url", "error", err)
return
}
req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, "db/"+proxy.ds.Database+"/"+proxy.proxyPath)
reqQueryVals.Add("u", proxy.ds.User)
reqQueryVals.Add("p", proxy.dataSourcesService.DecryptedPassword(proxy.ds))
reqQueryVals.Add("p", password)
req.URL.RawQuery = reqQueryVals.Encode()
case models.DS_INFLUXDB:
password, err := proxy.dataSourcesService.DecryptedPassword(req.Context(), proxy.ds)
if err != nil {
logger.Error("Error interpolating proxy url", "error", err)
return
}
req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, proxy.proxyPath)
req.URL.RawQuery = reqQueryVals.Encode()
if !proxy.ds.BasicAuth {
req.Header.Set(
"Authorization",
util.GetBasicAuthHeader(proxy.ds.User, proxy.dataSourcesService.DecryptedPassword(proxy.ds)),
util.GetBasicAuthHeader(proxy.ds.User, password),
)
}
default:
@@ -195,8 +203,13 @@ func (proxy *DataSourceProxy) director(req *http.Request) {
req.URL.Path = unescapedPath
if proxy.ds.BasicAuth {
password, err := proxy.dataSourcesService.DecryptedBasicAuthPassword(req.Context(), proxy.ds)
if err != nil {
logger.Error("Error interpolating proxy url", "error", err)
return
}
req.Header.Set("Authorization", util.GetBasicAuthHeader(proxy.ds.BasicAuthUser,
proxy.dataSourcesService.DecryptedBasicAuthPassword(proxy.ds)))
password))
}
dsAuth := req.Header.Get("X-DS-Authorization")
@@ -226,23 +239,23 @@ func (proxy *DataSourceProxy) director(req *http.Request) {
}
}
secureJsonData, err := proxy.secretsService.DecryptJsonData(req.Context(), proxy.ds.SecureJsonData)
if err != nil {
logger.Error("Error interpolating proxy url", "error", err)
return
}
if proxy.matchedRoute != nil {
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, DSInfo{
decryptedValues, err := proxy.dataSourcesService.DecryptedValues(req.Context(), proxy.ds)
if err != nil {
logger.Error("Error interpolating proxy url", "error", err)
return
}
ApplyRoute(req.Context(), req, proxy.proxyPath, proxy.matchedRoute, DSInfo{
ID: proxy.ds.Id,
Updated: proxy.ds.Updated,
JSONData: jsonData,
DecryptedSecureJSONData: secureJsonData,
DecryptedSecureJSONData: decryptedValues,
}, proxy.cfg)
}
if proxy.oAuthTokenService.IsOAuthPassThruEnabled(proxy.ds) {
if token := proxy.oAuthTokenService.GetCurrentOAuthToken(proxy.ctx.Req.Context(), proxy.ctx.SignedInUser); token != nil {
if token := proxy.oAuthTokenService.GetCurrentOAuthToken(req.Context(), proxy.ctx.SignedInUser); token != nil {
req.Header.Set("Authorization", fmt.Sprintf("%s %s", token.Type(), token.AccessToken))
idToken, ok := token.Extra("id_token").(string)

184
pkg/api/pluginproxy/ds_proxy_test.go
View File

@@ -3,6 +3,7 @@ package pluginproxy
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
@@ -24,6 +25,7 @@ import (
"github.com/grafana/grafana/pkg/services/oauthtoken"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/secrets/fakes"
"github.com/grafana/grafana/pkg/services/secrets/kvstore"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/web"
@@ -90,6 +92,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
})
setting.SecretKey = "password" //nolint:goconst
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
key, err := secretsService.Encrypt(context.Background(), []byte("123"), secrets.WithoutScope())
require.NoError(t, err)
@@ -128,9 +131,9 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("When matching route path", func(t *testing.T) {
ctx, req := setUp()
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider,
&oauthtoken.Service{}, dsService, tracer, secretsService)
&oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
proxy.matchedRoute = routes[0]
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg)
@@ -141,8 +144,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("When matching route path and has dynamic url", func(t *testing.T) {
ctx, req := setUp()
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/common/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/common/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
proxy.matchedRoute = routes[3]
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg)
@@ -153,8 +156,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("When matching route path with no url", func(t *testing.T) {
ctx, req := setUp()
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
proxy.matchedRoute = routes[4]
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg)
@@ -164,8 +167,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("When matching route path and has dynamic body", func(t *testing.T) {
ctx, req := setUp()
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/body", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/body", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
proxy.matchedRoute = routes[5]
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg)
@@ -178,8 +181,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("Validating request", func(t *testing.T) {
t.Run("plugin route with valid role", func(t *testing.T) {
ctx, _ := setUp()
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
err = proxy.validateRequest()
require.NoError(t, err)
@@ -187,8 +190,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("plugin route with admin role and user is editor", func(t *testing.T) {
ctx, _ := setUp()
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
err = proxy.validateRequest()
require.Error(t, err)
@@ -197,8 +200,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("plugin route with admin role and user is admin", func(t *testing.T) {
ctx, _ := setUp()
ctx.SignedInUser.OrgRole = models.ROLE_ADMIN
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
err = proxy.validateRequest()
require.NoError(t, err)
@@ -242,6 +245,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
})
setting.SecretKey = "password"
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
key, err := secretsService.Encrypt(context.Background(), []byte("123"), secrets.WithoutScope())
require.NoError(t, err)
@@ -286,8 +290,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
},
}
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[0], dsInfo, cfg)
@@ -302,8 +306,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
req, err := http.NewRequest("GET", "http://localhost/asd", nil)
require.NoError(t, err)
client = newFakeHTTPClient(t, json2)
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken2", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken2", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[1], dsInfo, cfg)
@@ -319,8 +323,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
require.NoError(t, err)
client = newFakeHTTPClient(t, []byte{})
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[0], dsInfo, cfg)
@@ -340,9 +344,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
ds := &models.DataSource{Url: "htttp://graphite:8080", Type: models.DS_GRAPHITE}
ctx := &models.ReqContext{}
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{BuildVersion: "5.3.0"}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{BuildVersion: "5.3.0"}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
require.NoError(t, err)
@@ -366,9 +371,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
ctx := &models.ReqContext{}
var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
@@ -390,9 +396,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
ctx := &models.ReqContext{}
var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
requestURL, err := url.Parse("http://grafana.com/sub")
@@ -418,9 +425,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
ctx := &models.ReqContext{}
var pluginRoutes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, pluginRoutes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, pluginRoutes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
requestURL, err := url.Parse("http://grafana.com/sub")
@@ -441,9 +449,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
}
ctx := &models.ReqContext{}
var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
req.Header.Set("Origin", "grafana.com")
@@ -490,9 +499,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
}
var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &mockAuthToken, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &mockAuthToken, dsService, tracer)
require.NoError(t, err)
req, err = http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
require.NoError(t, err)
@@ -543,24 +553,25 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
})
t.Run("When proxying data source proxy should handle authentication", func(t *testing.T) {
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
tests := []*testCase{
createAuthTest(t, secretsService, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, false),
createAuthTest(t, secretsService, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, true),
createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, true),
createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, false),
createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, true),
createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, false),
createAuthTest(t, secretsStore, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, false),
createAuthTest(t, secretsStore, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, true),
createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, true),
createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, false),
createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, true),
createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, false),
// These two should be enough for any other datasource at the moment. Proxy has special handling
// only for Influx, others have the same path and only BasicAuth. Non BasicAuth datasources
// do not go through proxy but through TSDB API which is not tested here.
createAuthTest(t, secretsService, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, false),
createAuthTest(t, secretsService, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, true),
createAuthTest(t, secretsStore, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, false),
createAuthTest(t, secretsStore, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, true),
}
for _, test := range tests {
runDatasourceAuthTest(t, secretsService, cfg, test)
runDatasourceAuthTest(t, secretsService, secretsStore, cfg, test)
}
})
}
@@ -624,9 +635,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
t.Run("When response header Set-Cookie is not set should remove proxied Set-Cookie header", func(t *testing.T) {
ctx, ds := setUp(t)
var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
proxy.HandleRequest()
@@ -642,9 +654,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
},
})
var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
proxy.HandleRequest()
@@ -656,9 +669,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
t.Run("When response should set Content-Security-Policy header", func(t *testing.T) {
ctx, ds := setUp(t)
var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
proxy.HandleRequest()
@@ -678,9 +692,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
},
})
var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
proxy.HandleRequest()
@@ -703,9 +718,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
ctx.Req = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil)
var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
proxy.HandleRequest()
@@ -727,9 +743,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
ctx.Req = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil)
var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
proxy.HandleRequest()
@@ -752,9 +769,10 @@ func TestNewDataSourceProxy_InvalidURL(t *testing.T) {
tracer, err := tracing.InitializeTracerForTest()
require.NoError(t, err)
var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
_, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
_, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer)
require.Error(t, err)
assert.True(t, strings.HasPrefix(err.Error(), `validation of data source URL "://host/root" failed`))
}
@@ -773,9 +791,10 @@ func TestNewDataSourceProxy_ProtocolLessURL(t *testing.T) {
require.NoError(t, err)
var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
_, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
_, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
}
@@ -816,9 +835,10 @@ func TestNewDataSourceProxy_MSSQL(t *testing.T) {
}
var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
p, err := NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
p, err := NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer)
if tc.err == nil {
require.NoError(t, err)
assert.Equal(t, &url.URL{
@@ -843,9 +863,10 @@ func getDatasourceProxiedRequest(t *testing.T, ctx *models.ReqContext, cfg *sett
require.NoError(t, err)
var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
require.NoError(t, err)
@@ -897,15 +918,15 @@ const (
authCheckHeader = "header"
)
func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string, url string, authType string, authCheck string, useSecureJsonData bool) *testCase {
ctx := context.Background()
func createAuthTest(t *testing.T, secretsStore kvstore.SecretsKVStore, dsType string, url string, authType string, authCheck string, useSecureJsonData bool) *testCase {
// Basic user:password
base64AuthHeader := "Basic dXNlcjpwYXNzd29yZA=="
test := &testCase{
datasource: &models.DataSource{
Id: 1,
OrgId: 1,
Name: fmt.Sprintf("%s,%s,%s,%s,%t", dsType, url, authType, authCheck, useSecureJsonData),
Type: dsType,
JsonData: simplejson.New(),
Url: url,
@@ -917,11 +938,13 @@ func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string,
message = fmt.Sprintf("%v should add username and password", dsType)
test.datasource.User = "user"
if useSecureJsonData {
test.datasource.SecureJsonData, err = secretsService.EncryptJsonData(
ctx,
map[string]string{
"password": "password",
}, secrets.WithoutScope())
secureJsonData, err := json.Marshal(map[string]string{
"password": "password",
})
require.NoError(t, err)
err = secretsStore.Set(context.Background(), test.datasource.OrgId, test.datasource.Name, "datasource", string(secureJsonData))
require.NoError(t, err)
} else {
test.datasource.Password = "password"
}
@@ -930,11 +953,13 @@ func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string,
test.datasource.BasicAuth = true
test.datasource.BasicAuthUser = "user"
if useSecureJsonData {
test.datasource.SecureJsonData, err = secretsService.EncryptJsonData(
ctx,
map[string]string{
"basicAuthPassword": "password",
}, secrets.WithoutScope())
secureJsonData, err := json.Marshal(map[string]string{
"basicAuthPassword": "password",
})
require.NoError(t, err)
err = secretsStore.Set(context.Background(), test.datasource.OrgId, test.datasource.Name, "datasource", string(secureJsonData))
require.NoError(t, err)
} else {
test.datasource.BasicAuthPassword = "password"
}
@@ -962,14 +987,14 @@ func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string,
return test
}
func runDatasourceAuthTest(t *testing.T, secretsService secrets.Service, cfg *setting.Cfg, test *testCase) {
func runDatasourceAuthTest(t *testing.T, secretsService secrets.Service, secretsStore kvstore.SecretsKVStore, cfg *setting.Cfg, test *testCase) {
ctx := &models.ReqContext{}
tracer, err := tracing.InitializeTracerForTest()
require.NoError(t, err)
var routes []*plugins.Route
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(test.datasource, routes, ctx, "", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(test.datasource, routes, ctx, "", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
@@ -1010,9 +1035,10 @@ func Test_PathCheck(t *testing.T) {
return ctx, req
}
ctx, _ := setUp()
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(&models.DataSource{}, routes, ctx, "b", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService)
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(&models.DataSource{}, routes, ctx, "b", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err)
require.Nil(t, proxy.validateRequest())

16
pkg/expr/service.go
View File

@@ -7,7 +7,7 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/setting"
)
@@ -36,16 +36,16 @@ func IsDataSource(uid string) bool {
// Service is service representation for expression handling.
type Service struct {
cfg *setting.Cfg
dataService backend.QueryDataHandler
secretsService secrets.Service
cfg *setting.Cfg
dataService backend.QueryDataHandler
dataSourceService datasources.DataSourceService
}
func ProvideService(cfg *setting.Cfg, pluginClient plugins.Client, secretsService secrets.Service) *Service {
func ProvideService(cfg *setting.Cfg, pluginClient plugins.Client, dataSourceService datasources.DataSourceService) *Service {
return &Service{
cfg: cfg,
dataService: pluginClient,
secretsService: secretsService,
cfg: cfg,
dataService: pluginClient,
dataSourceService: dataSourceService,
}
}

11
pkg/expr/service_test.go
View File

@@ -11,8 +11,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/secrets/fakes"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
datasources "github.com/grafana/grafana/pkg/services/datasources/fakes"
"github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/require"
)
@@ -28,12 +27,10 @@ func TestService(t *testing.T) {
cfg := setting.NewCfg()
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
s := Service{
cfg: cfg,
dataService: me,
secretsService: secretsService,
cfg: cfg,
dataService: me,
dataSourceService: &datasources.FakeDataSourceService{},
}
queries := []Query{

6
pkg/expr/transform.go
View File

@@ -126,9 +126,9 @@ func hiddenRefIDs(queries []Query) (map[string]struct{}, error) {
return hidden, nil
}
func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(map[string][]byte) map[string]string {
return func(m map[string][]byte) map[string]string {
decryptedJsonData, err := s.secretsService.DecryptJsonData(ctx, m)
func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string {
return func(ds *models.DataSource) map[string]string {
decryptedJsonData, err := s.dataSourceService.DecryptedValues(ctx, ds)
if err != nil {
logger.Error("Failed to decrypt secure json data", "error", err)
}

2
pkg/infra/usagestats/statscollector/prometheus_flavor.go
View File

@@ -60,7 +60,7 @@ func (s *Service) detectPrometheusVariant(ctx context.Context, ds *models.DataSo
} `json:"data"`
}
c, err := s.datasources.GetHTTPTransport(ds, s.httpClientProvider)
c, err := s.datasources.GetHTTPTransport(ctx, ds, s.httpClientProvider)
if err != nil {
s.log.Error("Failed to get HTTP client for Prometheus data source", "error", err)
return "", err

2
pkg/infra/usagestats/statscollector/service_test.go
View File

@@ -395,6 +395,6 @@ func (s mockDatasourceService) GetDataSourcesByType(ctx context.Context, query *
return nil
}
func (s mockDatasourceService) GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) {
func (s mockDatasourceService) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) {
return provider.GetTransport()
}

4
pkg/plugins/adapters/adapters.go
View File

@@ -9,7 +9,7 @@ import (
)
// ModelToInstanceSettings converts a models.DataSource to a backend.DataSourceInstanceSettings.
func ModelToInstanceSettings(ds *models.DataSource, decryptFn func(map[string][]byte) map[string]string,
func ModelToInstanceSettings(ds *models.DataSource, decryptFn func(ds *models.DataSource) map[string]string,
) (*backend.DataSourceInstanceSettings, error) {
var jsonDataBytes json.RawMessage
if ds.JsonData != nil {
@@ -30,7 +30,7 @@ func ModelToInstanceSettings(ds *models.DataSource, decryptFn func(map[string][]
BasicAuthEnabled: ds.BasicAuth,
BasicAuthUser: ds.BasicAuthUser,
JSONData: jsonDataBytes,
DecryptedSecureJSONData: decryptFn(ds.SecureJsonData),
DecryptedSecureJSONData: decryptFn(ds),
Updated: ds.Updated,
}, nil
}

15
pkg/plugins/plugincontext/plugincontext.go
View File

@@ -15,18 +15,17 @@ import (
"github.com/grafana/grafana/pkg/plugins/adapters"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/pluginsettings"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/util/errutil"
)
func ProvideService(cacheService *localcache.CacheService, pluginStore plugins.Store,
dataSourceCache datasources.CacheService, secretsService secrets.Service,
dataSourceCache datasources.CacheService, dataSourceService datasources.DataSourceService,
pluginSettingsService pluginsettings.Service) *Provider {
return &Provider{
cacheService: cacheService,
pluginStore: pluginStore,
dataSourceCache: dataSourceCache,
secretsService: secretsService,
dataSourceService: dataSourceService,
pluginSettingsService: pluginSettingsService,
logger: log.New("plugincontext"),
}
@@ -36,7 +35,7 @@ type Provider struct {
cacheService *localcache.CacheService
pluginStore plugins.Store
dataSourceCache datasources.CacheService
secretsService secrets.Service
dataSourceService datasources.DataSourceService
pluginSettingsService pluginsettings.Service
logger log.Logger
}
@@ -87,7 +86,7 @@ func (p *Provider) Get(ctx context.Context, pluginID string, datasourceUID strin
if err != nil {
return pc, false, errutil.Wrap("Failed to get datasource", err)
}
datasourceSettings, err := adapters.ModelToInstanceSettings(ds, p.decryptSecureJsonDataFn())
datasourceSettings, err := adapters.ModelToInstanceSettings(ds, p.decryptSecureJsonDataFn(ctx))
if err != nil {
return pc, false, errutil.Wrap("Failed to convert datasource", err)
}
@@ -122,9 +121,9 @@ func (p *Provider) getCachedPluginSettings(ctx context.Context, pluginID string,
return ps, nil
}
func (p *Provider) decryptSecureJsonDataFn() func(map[string][]byte) map[string]string {
return func(m map[string][]byte) map[string]string {
decryptedJsonData, err := p.secretsService.DecryptJsonData(context.Background(), m)
func (p *Provider) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string {
return func(ds *models.DataSource) map[string]string {
decryptedJsonData, err := p.dataSourceService.DecryptedValues(ctx, ds)
if err != nil {
p.logger.Error("Failed to decrypt secure json data", "error", err)
}

2
pkg/server/wire.go
View File

@@ -79,6 +79,7 @@ import (
"github.com/grafana/grafana/pkg/services/searchV2"
"github.com/grafana/grafana/pkg/services/secrets"
secretsDatabase "github.com/grafana/grafana/pkg/services/secrets/database"
secretsStore "github.com/grafana/grafana/pkg/services/secrets/kvstore"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/services/serviceaccounts"
serviceaccountsmanager "github.com/grafana/grafana/pkg/services/serviceaccounts/manager"
@@ -239,6 +240,7 @@ var wireBasicSet = wire.NewSet(
wire.Bind(new(alerting.DashAlertExtractor), new(*alerting.DashAlertExtractorService)),
comments.ProvideService,
guardian.ProvideService,
secretsStore.ProvideService,
avatar.ProvideAvatarCacheServer,
authproxy.ProvideAuthProxy,
statscollector.ProvideService,

2
pkg/services/datasourceproxy/datasourceproxy.go
View File

@@ -115,7 +115,7 @@ func (p *DataSourceProxyService) proxyDatasourceRequest(c *models.ReqContext, ds
proxyPath := getProxyPath(c)
proxy, err := pluginproxy.NewDataSourceProxy(ds, plugin.Routes, c, proxyPath, p.Cfg, p.HTTPClientProvider,
p.OAuthTokenService, p.DataSourcesService, p.tracer, p.secretsService)
p.OAuthTokenService, p.DataSourcesService, p.tracer)
if err != nil {
if errors.Is(err, datasource.URLValidationError{}) {
c.JsonApiErr(http.StatusBadRequest, fmt.Sprintf("Invalid data source URL: %q", ds.Url), err)

10
pkg/services/datasources/datasources.go
View File

@@ -33,23 +33,23 @@ type DataSourceService interface {
GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error
// GetHTTPTransport gets a datasource specific HTTP transport.
GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error)
GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error)
// DecryptedValues decrypts the encrypted secureJSONData of the provided datasource and
// returns the decrypted values.
DecryptedValues(ds *models.DataSource) map[string]string
DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error)
// DecryptedValue decrypts the encrypted datasource secureJSONData identified by key
// and returns the decryped value.
DecryptedValue(ds *models.DataSource, key string) (string, bool)
DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error)
// DecryptedBasicAuthPassword decrypts the encrypted datasource basic authentication
// password and returns the decryped value.
DecryptedBasicAuthPassword(ds *models.DataSource) string
DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error)
// DecryptedPassword decrypts the encrypted datasource password and returns the
// decryped value.
DecryptedPassword(ds *models.DataSource) string
DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error)
}
// CacheService interface for retrieving a cached datasource.

3
pkg/services/datasources/fake_cache_service.go → pkg/services/datasources/fakes/fake_cache_service.go
View File

@@ -4,13 +4,14 @@ import (
"context"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/datasources"
)
type FakeCacheService struct {
DataSources []*models.DataSource
}
var _ CacheService = &FakeCacheService{}
var _ datasources.CacheService = &FakeCacheService{}
func (c *FakeCacheService) GetDatasource(ctx context.Context, datasourceID int64, user *models.SignedInUser, skipCache bool) (*models.DataSource, error) {
for _, datasource := range c.DataSources {

124
pkg/services/datasources/fakes/fake_datasource_service.go Normal file
View File

@@ -0,0 +1,124 @@
package datasources
import (
"context"
"net/http"
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/datasources"
)
type FakeDataSourceService struct {
lastId int64
DataSources []*models.DataSource
}
var _ datasources.DataSourceService = &FakeDataSourceService{}
func (s *FakeDataSourceService) GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error {
for _, datasource := range s.DataSources {
idMatch := query.Id != 0 && query.Id == datasource.Id
uidMatch := query.Uid != "" && query.Uid == datasource.Uid
nameMatch := query.Name != "" && query.Name == datasource.Name
if idMatch || nameMatch || uidMatch {
query.Result = datasource
return nil
}
}
return models.ErrDataSourceNotFound
}
func (s *FakeDataSourceService) GetDataSources(ctx context.Context, query *models.GetDataSourcesQuery) error {
for _, datasource := range s.DataSources {
orgMatch := query.OrgId != 0 && query.OrgId == datasource.OrgId
if orgMatch {
query.Result = append(query.Result, datasource)
}
}
return nil
}
func (s *FakeDataSourceService) GetDataSourcesByType(ctx context.Context, query *models.GetDataSourcesByTypeQuery) error {
for _, datasource := range s.DataSources {
typeMatch := query.Type != "" && query.Type == datasource.Type
if typeMatch {
query.Result = append(query.Result, datasource)
}
}
return nil
}
func (s *FakeDataSourceService) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCommand) error {
if s.lastId == 0 {
s.lastId = int64(len(s.DataSources) - 1)
}
cmd.Result = &models.DataSource{
Id: s.lastId + 1,
Name: cmd.Name,
Type: cmd.Type,
Uid: cmd.Uid,
OrgId: cmd.OrgId,
}
s.DataSources = append(s.DataSources, cmd.Result)
return nil
}
func (s *FakeDataSourceService) DeleteDataSource(ctx context.Context, cmd *models.DeleteDataSourceCommand) error {
for i, datasource := range s.DataSources {
idMatch := cmd.ID != 0 && cmd.ID == datasource.Id
uidMatch := cmd.UID != "" && cmd.UID == datasource.Uid
nameMatch := cmd.Name != "" && cmd.Name == datasource.Name
if idMatch || nameMatch || uidMatch {
s.DataSources = append(s.DataSources[:i], s.DataSources[i+1:]...)
return nil
}
}
return models.ErrDataSourceNotFound
}
func (s *FakeDataSourceService) UpdateDataSource(ctx context.Context, cmd *models.UpdateDataSourceCommand) error {
for _, datasource := range s.DataSources {
idMatch := cmd.Id != 0 && cmd.Id == datasource.Id
uidMatch := cmd.Uid != "" && cmd.Uid == datasource.Uid
nameMatch := cmd.Name != "" && cmd.Name == datasource.Name
if idMatch || nameMatch || uidMatch {
if cmd.Name != "" {
datasource.Name = cmd.Name
}
return nil
}
}
return models.ErrDataSourceNotFound
}
func (s *FakeDataSourceService) GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error {
return nil
}
func (s *FakeDataSourceService) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) {
rt, err := provider.GetTransport(sdkhttpclient.Options{})
if err != nil {
return nil, err
}
return rt, nil
}
func (s *FakeDataSourceService) DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) {
values := make(map[string]string)
return values, nil
}
func (s *FakeDataSourceService) DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error) {
return "", false, nil
}
func (s *FakeDataSourceService) DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error) {
return "", nil
}
func (s *FakeDataSourceService) DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error) {
return "", nil
}

237
pkg/services/datasources/service/datasource_service.go
View File

@@ -3,6 +3,7 @@ package service
import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"net/http"
"net/url"
@@ -23,20 +24,21 @@ import (
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/secrets/kvstore"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
)
type Service struct {
SQLStore *sqlstore.SQLStore
SecretsStore kvstore.SecretsKVStore
SecretsService secrets.Service
cfg *setting.Cfg
features featuremgmt.FeatureToggles
permissionsService accesscontrol.PermissionsService
ac accesscontrol.AccessControl
ptc proxyTransportCache
dsDecryptionCache secureJSONDecryptionCache
ptc proxyTransportCache
}
type proxyTransportCache struct {
@@ -49,29 +51,17 @@ type cachedRoundTripper struct {
roundTripper http.RoundTripper
}
type secureJSONDecryptionCache struct {
cache map[int64]cachedDecryptedJSON
sync.Mutex
}
type cachedDecryptedJSON struct {
updated time.Time
json map[string]string
}
func ProvideService(
store *sqlstore.SQLStore, secretsService secrets.Service, cfg *setting.Cfg, features featuremgmt.FeatureToggles,
ac accesscontrol.AccessControl, permissionsServices accesscontrol.PermissionsServices,
store *sqlstore.SQLStore, secretsService secrets.Service, secretsStore kvstore.SecretsKVStore, cfg *setting.Cfg,
features featuremgmt.FeatureToggles, ac accesscontrol.AccessControl, permissionsServices accesscontrol.PermissionsServices,
) *Service {
s := &Service{
SQLStore: store,
SecretsStore: secretsStore,
SecretsService: secretsService,
ptc: proxyTransportCache{
cache: make(map[int64]cachedRoundTripper),
},
dsDecryptionCache: secureJSONDecryptionCache{
cache: make(map[int64]cachedDecryptedJSON),
},
cfg: cfg,
features: features,
permissionsService: permissionsServices.GetDataSourceService(),
@@ -90,6 +80,8 @@ type DataSourceRetriever interface {
GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error
}
const secretType = "datasource"
// NewNameScopeResolver provides an AttributeScopeResolver able to
// translate a scope prefixed with "datasources:name:" into an uid based scope.
func NewNameScopeResolver(db DataSourceRetriever) (string, accesscontrol.AttributeScopeResolveFunc) {
@@ -155,12 +147,17 @@ func (s *Service) GetDataSourcesByType(ctx context.Context, query *models.GetDat
func (s *Service) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCommand) error {
var err error
cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope())
if err := s.SQLStore.AddDataSource(ctx, cmd); err != nil {
return err
}
secret, err := json.Marshal(cmd.SecureJsonData)
if err != nil {
return err
}
if err := s.SQLStore.AddDataSource(ctx, cmd); err != nil {
err = s.SecretsStore.Set(ctx, cmd.OrgId, cmd.Name, secretType, string(secret))
if err != nil {
return err
}
@@ -186,25 +183,50 @@ func (s *Service) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCo
}
func (s *Service) DeleteDataSource(ctx context.Context, cmd *models.DeleteDataSourceCommand) error {
return s.SQLStore.DeleteDataSource(ctx, cmd)
err := s.SQLStore.DeleteDataSource(ctx, cmd)
if err != nil {
return err
}
return s.SecretsStore.Del(ctx, cmd.OrgID, cmd.Name, secretType)
}
func (s *Service) UpdateDataSource(ctx context.Context, cmd *models.UpdateDataSourceCommand) error {
var err error
cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope())
secret, err := json.Marshal(cmd.SecureJsonData)
if err != nil {
return err
}
return s.SQLStore.UpdateDataSource(ctx, cmd)
query := &models.GetDataSourceQuery{
Id: cmd.Id,
OrgId: cmd.OrgId,
}
err = s.SQLStore.GetDataSource(ctx, query)
if err != nil {
return err
}
err = s.SQLStore.UpdateDataSource(ctx, cmd)
if err != nil {
return err
}
if query.Result.Name != cmd.Name {
err = s.SecretsStore.Rename(ctx, cmd.OrgId, query.Result.Name, secretType, cmd.Name)
if err != nil {
return err
}
}
return s.SecretsStore.Set(ctx, cmd.OrgId, cmd.Name, secretType, string(secret))
}
func (s *Service) GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error {
return s.SQLStore.GetDefaultDataSource(ctx, query)
}
func (s *Service) GetHTTPClient(ds *models.DataSource, provider httpclient.Provider) (*http.Client, error) {
transport, err := s.GetHTTPTransport(ds, provider)
func (s *Service) GetHTTPClient(ctx context.Context, ds *models.DataSource, provider httpclient.Provider) (*http.Client, error) {
transport, err := s.GetHTTPTransport(ctx, ds, provider)
if err != nil {
return nil, err
}
@@ -215,7 +237,7 @@ func (s *Service) GetHTTPClient(ds *models.DataSource, provider httpclient.Provi
}, nil
}
func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider,
func (s *Service) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider,
customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) {
s.ptc.Lock()
defer s.ptc.Unlock()
@@ -224,7 +246,7 @@ func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Pr
return t.roundTripper, nil
}
opts, err := s.httpClientOptions(ds)
opts, err := s.httpClientOptions(ctx, ds)
if err != nil {
return nil, err
}
@@ -244,58 +266,84 @@ func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Pr
return rt, nil
}
func (s *Service) GetTLSConfig(ds *models.DataSource, httpClientProvider httpclient.Provider) (*tls.Config, error) {
opts, err := s.httpClientOptions(ds)
func (s *Service) GetTLSConfig(ctx context.Context, ds *models.DataSource, httpClientProvider httpclient.Provider) (*tls.Config, error) {
opts, err := s.httpClientOptions(ctx, ds)
if err != nil {
return nil, err
}
return httpClientProvider.GetTLSConfig(*opts)
}
func (s *Service) DecryptedValues(ds *models.DataSource) map[string]string {
s.dsDecryptionCache.Lock()
defer s.dsDecryptionCache.Unlock()
if item, present := s.dsDecryptionCache.cache[ds.Id]; present && ds.Updated.Equal(item.updated) {
return item.json
}
json, err := s.SecretsService.DecryptJsonData(context.Background(), ds.SecureJsonData)
func (s *Service) DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) {
decryptedValues := make(map[string]string)
secret, exist, err := s.SecretsStore.Get(ctx, ds.OrgId, ds.Name, secretType)
if err != nil {
return map[string]string{}
return nil, err
}
s.dsDecryptionCache.cache[ds.Id] = cachedDecryptedJSON{
updated: ds.Updated,
json: json,
if exist {
err := json.Unmarshal([]byte(secret), &decryptedValues)
if err != nil {
return nil, err
}
} else if len(ds.SecureJsonData) > 0 {
decryptedValues, err = s.MigrateSecrets(ctx, ds)
if err != nil {
return nil, err
}
}
return json
return decryptedValues, nil
}
func (s *Service) DecryptedValue(ds *models.DataSource, key string) (string, bool) {
value, exists := s.DecryptedValues(ds)[key]
return value, exists
}
func (s *Service) DecryptedBasicAuthPassword(ds *models.DataSource) string {
if value, ok := s.DecryptedValue(ds, "basicAuthPassword"); ok {
return value
func (s *Service) MigrateSecrets(ctx context.Context, ds *models.DataSource) (map[string]string, error) {
secureJsonData, err := s.SecretsService.DecryptJsonData(ctx, ds.SecureJsonData)
if err != nil {
return nil, err
}
return ds.BasicAuthPassword
}
func (s *Service) DecryptedPassword(ds *models.DataSource) string {
if value, ok := s.DecryptedValue(ds, "password"); ok {
return value
jsonData, err := json.Marshal(secureJsonData)
if err != nil {
return nil, err
}
return ds.Password
err = s.SecretsStore.Set(ctx, ds.OrgId, ds.Name, secretType, string(jsonData))
return secureJsonData, err
}
func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Options, error) {
tlsOptions := s.dsTLSOptions(ds)
func (s *Service) DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error) {
values, err := s.DecryptedValues(ctx, ds)
if err != nil {
return "", false, err
}
value, exists := values[key]
return value, exists, nil
}
func (s *Service) DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error) {
value, ok, err := s.DecryptedValue(ctx, ds, "basicAuthPassword")
if ok {
return value, nil
}
return ds.BasicAuthPassword, err
}
func (s *Service) DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error) {
value, ok, err := s.DecryptedValue(ctx, ds, "password")
if ok {
return value, nil
}
return ds.Password, err
}
func (s *Service) httpClientOptions(ctx context.Context, ds *models.DataSource) (*sdkhttpclient.Options, error) {
tlsOptions, err := s.dsTLSOptions(ctx, ds)
if err != nil {
return nil, err
}
timeouts := &sdkhttpclient.TimeoutOptions{
Timeout: s.getTimeout(ds),
DialTimeout: sdkhttpclient.DefaultTimeoutOptions.DialTimeout,
@@ -307,9 +355,15 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio
MaxIdleConnsPerHost: sdkhttpclient.DefaultTimeoutOptions.MaxIdleConnsPerHost,
IdleConnTimeout: sdkhttpclient.DefaultTimeoutOptions.IdleConnTimeout,
}
decryptedValues, err := s.DecryptedValues(ctx, ds)
if err != nil {
return nil, err
}
opts := &sdkhttpclient.Options{
Timeouts: timeouts,
Headers: s.getCustomHeaders(ds.JsonData, s.DecryptedValues(ds)),
Headers: s.getCustomHeaders(ds.JsonData, decryptedValues),
Labels: map[string]string{
"datasource_name": ds.Name,
"datasource_uid": ds.Uid,
@@ -320,22 +374,30 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio
if ds.JsonData != nil {
opts.CustomOptions = ds.JsonData.MustMap()
}
if ds.BasicAuth {
password, err := s.DecryptedBasicAuthPassword(ctx, ds)
if err != nil {
return opts, err
}
opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{
User: ds.BasicAuthUser,
Password: s.DecryptedBasicAuthPassword(ds),
Password: password,
}
} else if ds.User != "" {
password, err := s.DecryptedPassword(ctx, ds)
if err != nil {
return opts, err
}
opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{
User: ds.User,
Password: s.DecryptedPassword(ds),
Password: password,
}
}
// Azure authentication
if ds.JsonData != nil && s.features.IsEnabled(featuremgmt.FlagHttpclientproviderAzureAuth) {
credentials, err := azcredentials.FromDatasourceData(ds.JsonData.MustMap(), s.DecryptedValues(ds))
credentials, err := azcredentials.FromDatasourceData(ds.JsonData.MustMap(), decryptedValues)
if err != nil {
err = fmt.Errorf("invalid Azure credentials: %s", err)
return nil, err
@@ -371,19 +433,27 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio
Profile: ds.JsonData.Get("sigV4Profile").MustString(),
}
if val, exists := s.DecryptedValue(ds, "sigV4AccessKey"); exists {
opts.SigV4.AccessKey = val
if val, exists, err := s.DecryptedValue(ctx, ds, "sigV4AccessKey"); err == nil {
if exists {
opts.SigV4.AccessKey = val
}
} else {
return opts, err
}
if val, exists := s.DecryptedValue(ds, "sigV4SecretKey"); exists {
opts.SigV4.SecretKey = val
if val, exists, err := s.DecryptedValue(ctx, ds, "sigV4SecretKey"); err == nil {
if exists {
opts.SigV4.SecretKey = val
}
} else {
return opts, err
}
}
return opts, nil
}
func (s *Service) dsTLSOptions(ds *models.DataSource) sdkhttpclient.TLSOptions {
func (s *Service) dsTLSOptions(ctx context.Context, ds *models.DataSource) (sdkhttpclient.TLSOptions, error) {
var tlsSkipVerify, tlsClientAuth, tlsAuthWithCACert bool
var serverName string
@@ -401,22 +471,35 @@ func (s *Service) dsTLSOptions(ds *models.DataSource) sdkhttpclient.TLSOptions {
if tlsClientAuth || tlsAuthWithCACert {
if tlsAuthWithCACert {
if val, exists := s.DecryptedValue(ds, "tlsCACert"); exists && len(val) > 0 {
opts.CACertificate = val
if val, exists, err := s.DecryptedValue(ctx, ds, "tlsCACert"); err == nil {
if exists && len(val) > 0 {
opts.CACertificate = val
}
} else {
return opts, err
}
}
if tlsClientAuth {
if val, exists := s.DecryptedValue(ds, "tlsClientCert"); exists && len(val) > 0 {
opts.ClientCertificate = val
if val, exists, err := s.DecryptedValue(ctx, ds, "tlsClientCert"); err == nil {
fmt.Print("\n\n\n\n", val, exists, err, "\n\n\n\n")
if exists && len(val) > 0 {
opts.ClientCertificate = val
}
} else {
return opts, err
}
if val, exists := s.DecryptedValue(ds, "tlsClientKey"); exists && len(val) > 0 {
opts.ClientKey = val
if val, exists, err := s.DecryptedValue(ctx, ds, "tlsClientKey"); err == nil {
if exists && len(val) > 0 {
opts.ClientKey = val
}
} else {
return opts, err
}
}
}
return opts
return opts, nil
}
func (s *Service) getTimeout(ds *models.DataSource) time.Duration {

278
pkg/services/datasources/service/datasource_service_test.go
View File

@@ -2,6 +2,7 @@ package service
import (
"context"
encJson "encoding/json"
"io/ioutil"
"net/http"
"net/http/httptest"
@@ -10,6 +11,7 @@ import (
"github.com/grafana/grafana-azure-sdk-go/azsettings"
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
@@ -17,59 +19,13 @@ import (
"github.com/grafana/grafana/pkg/services/accesscontrol"
acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/secrets/database"
"github.com/grafana/grafana/pkg/services/secrets/fakes"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/services/secrets/kvstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestService(t *testing.T) {
cfg := &setting.Cfg{}
sqlStore := sqlstore.InitTestDB(t)
origSecret := setting.SecretKey
setting.SecretKey = "datasources_service_test"
t.Cleanup(func() {
setting.SecretKey = origSecret
})
secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(sqlStore))
s := ProvideService(sqlStore, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New().WithDisabled(), acmock.NewPermissionsServicesMock())
var ds *models.DataSource
t.Run("create datasource should encrypt the secure json data", func(t *testing.T) {
ctx := context.Background()
sjd := map[string]string{"password": "12345"}
cmd := models.AddDataSourceCommand{SecureJsonData: sjd}
err := s.AddDataSource(ctx, &cmd)
require.NoError(t, err)
ds = cmd.Result
decrypted, err := s.SecretsService.DecryptJsonData(ctx, ds.SecureJsonData)
require.NoError(t, err)
require.Equal(t, sjd, decrypted)
})
t.Run("update datasource should encrypt the secure json data", func(t *testing.T) {
ctx := context.Background()
sjd := map[string]string{"password": "678910"}
cmd := models.UpdateDataSourceCommand{Id: ds.Id, OrgId: ds.OrgId, SecureJsonData: sjd}
err := s.UpdateDataSource(ctx, &cmd)
require.NoError(t, err)
decrypted, err := s.SecretsService.DecryptJsonData(ctx, cmd.Result.SecureJsonData)
require.NoError(t, err)
require.Equal(t, sjd, decrypted)
})
}
type dataSourceMockRetriever struct {
res []*models.DataSource
}
@@ -237,15 +193,16 @@ func TestService_GetHttpTransport(t *testing.T) {
Type: "Kubernetes",
}
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
rt1, err := dsService.GetHTTPTransport(&ds, provider)
rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err)
require.NotNil(t, rt1)
tr1 := configuredTransport
rt2, err := dsService.GetHTTPTransport(&ds, provider)
rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err)
require.NotNil(t, rt2)
tr2 := configuredTransport
@@ -270,21 +227,19 @@ func TestService_GetHttpTransport(t *testing.T) {
json := simplejson.New()
json.Set("tlsAuthWithCACert", true)
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
tlsCaCert, err := secretsService.Encrypt(context.Background(), []byte(caCert), secrets.WithoutScope())
require.NoError(t, err)
dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
ds := models.DataSource{
Id: 1,
Url: "http://k8s:8001",
Type: "Kubernetes",
SecureJsonData: map[string][]byte{"tlsCACert": tlsCaCert},
SecureJsonData: map[string][]byte{"tlsCACert": []byte(caCert)},
Updated: time.Now().Add(-2 * time.Minute),
}
rt1, err := dsService.GetHTTPTransport(&ds, provider)
rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NotNil(t, rt1)
require.NoError(t, err)
@@ -298,7 +253,7 @@ func TestService_GetHttpTransport(t *testing.T) {
ds.SecureJsonData = map[string][]byte{}
ds.Updated = time.Now()
rt2, err := dsService.GetHTTPTransport(&ds, provider)
rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err)
require.NotNil(t, rt2)
tr2 := configuredTransport
@@ -320,27 +275,29 @@ func TestService_GetHttpTransport(t *testing.T) {
json := simplejson.New()
json.Set("tlsAuth", true)
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
tlsClientCert, err := secretsService.Encrypt(context.Background(), []byte(clientCert), secrets.WithoutScope())
require.NoError(t, err)
tlsClientKey, err := secretsService.Encrypt(context.Background(), []byte(clientKey), secrets.WithoutScope())
require.NoError(t, err)
dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
ds := models.DataSource{
Id: 1,
OrgId: 1,
Name: "kubernetes",
Url: "http://k8s:8001",
Type: "Kubernetes",
JsonData: json,
SecureJsonData: map[string][]byte{
"tlsClientCert": tlsClientCert,
"tlsClientKey": tlsClientKey,
},
}
rt, err := dsService.GetHTTPTransport(&ds, provider)
secureJsonData, err := encJson.Marshal(map[string]string{
"tlsClientCert": clientCert,
"tlsClientKey": clientKey,
})
require.NoError(t, err)
err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData))
require.NoError(t, err)
rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err)
require.NotNil(t, rt)
tr := configuredTransport
@@ -363,23 +320,28 @@ func TestService_GetHttpTransport(t *testing.T) {
json.Set("tlsAuthWithCACert", true)
json.Set("serverName", "server-name")
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
tlsCaCert, err := secretsService.Encrypt(context.Background(), []byte(caCert), secrets.WithoutScope())
require.NoError(t, err)
dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
ds := models.DataSource{
Id: 1,
OrgId: 1,
Name: "kubernetes",
Url: "http://k8s:8001",
Type: "Kubernetes",
JsonData: json,
SecureJsonData: map[string][]byte{
"tlsCACert": tlsCaCert,
},
}
rt, err := dsService.GetHTTPTransport(&ds, provider)
secureJsonData, err := encJson.Marshal(map[string]string{
"tlsCACert": caCert,
})
require.NoError(t, err)
err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData))
require.NoError(t, err)
rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err)
require.NotNil(t, rt)
tr := configuredTransport
@@ -400,8 +362,9 @@ func TestService_GetHttpTransport(t *testing.T) {
json := simplejson.New()
json.Set("tlsSkipVerify", true)
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
ds := models.DataSource{
Id: 1,
@@ -410,12 +373,12 @@ func TestService_GetHttpTransport(t *testing.T) {
JsonData: json,
}
rt1, err := dsService.GetHTTPTransport(&ds, provider)
rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err)
require.NotNil(t, rt1)
tr1 := configuredTransport
rt2, err := dsService.GetHTTPTransport(&ds, provider)
rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err)
require.NotNil(t, rt2)
tr2 := configuredTransport
@@ -431,20 +394,27 @@ func TestService_GetHttpTransport(t *testing.T) {
"httpHeaderName1": "Authorization",
})
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
encryptedData, err := secretsService.Encrypt(context.Background(), []byte(`Bearer xf5yhfkpsnmgo`), secrets.WithoutScope())
require.NoError(t, err)
dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
ds := models.DataSource{
Id: 1,
Url: "http://k8s:8001",
Type: "Kubernetes",
JsonData: json,
SecureJsonData: map[string][]byte{"httpHeaderValue1": encryptedData},
Id: 1,
OrgId: 1,
Name: "kubernetes",
Url: "http://k8s:8001",
Type: "Kubernetes",
JsonData: json,
}
secureJsonData, err := encJson.Marshal(map[string]string{
"httpHeaderValue1": "Bearer xf5yhfkpsnmgo",
})
require.NoError(t, err)
err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData))
require.NoError(t, err)
headers := dsService.getCustomHeaders(json, map[string]string{"httpHeaderValue1": "Bearer xf5yhfkpsnmgo"})
require.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"])
@@ -465,7 +435,7 @@ func TestService_GetHttpTransport(t *testing.T) {
// 2. Get HTTP transport from datasource which uses the test server as backend
ds.Url = backend.URL
rt, err := dsService.GetHTTPTransport(&ds, provider)
rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err)
require.NotNil(t, rt)
@@ -490,8 +460,9 @@ func TestService_GetHttpTransport(t *testing.T) {
"timeout": 19,
})
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
ds := models.DataSource{
Id: 1,
@@ -500,7 +471,7 @@ func TestService_GetHttpTransport(t *testing.T) {
JsonData: json,
}
client, err := dsService.GetHTTPClient(&ds, provider)
client, err := dsService.GetHTTPClient(context.Background(), &ds, provider)
require.NoError(t, err)
require.NotNil(t, client)
require.Equal(t, 19*time.Second, client.Timeout)
@@ -523,15 +494,16 @@ func TestService_GetHttpTransport(t *testing.T) {
json, err := simplejson.NewJson([]byte(`{ "sigV4Auth": true }`))
require.NoError(t, err)
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
ds := models.DataSource{
Type: models.DS_ES,
JsonData: json,
}
_, err = dsService.GetHTTPTransport(&ds, provider)
_, err = dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err)
require.NotNil(t, configuredOpts)
require.NotNil(t, configuredOpts.SigV4)
@@ -558,8 +530,9 @@ func TestService_getTimeout(t *testing.T) {
{jsonData: simplejson.NewFromAny(map[string]interface{}{"timeout": "2"}), expectedTimeout: 2 * time.Second},
}
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
for _, tc := range testCases {
ds := &models.DataSource{
@@ -569,86 +542,6 @@ func TestService_getTimeout(t *testing.T) {
}
}
func TestService_DecryptedValue(t *testing.T) {
cfg := &setting.Cfg{}
t.Run("When datasource hasn't been updated, encrypted JSON should be fetched from cache", func(t *testing.T) {
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
encryptedJsonData, err := secretsService.EncryptJsonData(
context.Background(),
map[string]string{
"password": "password",
}, secrets.WithoutScope())
require.NoError(t, err)
ds := models.DataSource{
Id: 1,
Type: models.DS_INFLUXDB_08,
JsonData: simplejson.New(),
User: "user",
SecureJsonData: encryptedJsonData,
}
// Populate cache
password, ok := dsService.DecryptedValue(&ds, "password")
require.True(t, ok)
require.Equal(t, "password", password)
encryptedJsonData, err = secretsService.EncryptJsonData(
context.Background(),
map[string]string{
"password": "",
}, secrets.WithoutScope())
require.NoError(t, err)
ds.SecureJsonData = encryptedJsonData
password, ok = dsService.DecryptedValue(&ds, "password")
require.True(t, ok)
require.Equal(t, "password", password)
})
t.Run("When datasource is updated, encrypted JSON should not be fetched from cache", func(t *testing.T) {
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
encryptedJsonData, err := secretsService.EncryptJsonData(
context.Background(),
map[string]string{
"password": "password",
}, secrets.WithoutScope())
require.NoError(t, err)
ds := models.DataSource{
Id: 1,
Type: models.DS_INFLUXDB_08,
JsonData: simplejson.New(),
User: "user",
SecureJsonData: encryptedJsonData,
}
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
// Populate cache
password, ok := dsService.DecryptedValue(&ds, "password")
require.True(t, ok)
require.Equal(t, "password", password)
ds.SecureJsonData, err = secretsService.EncryptJsonData(
context.Background(),
map[string]string{
"password": "",
}, secrets.WithoutScope())
ds.Updated = time.Now()
require.NoError(t, err)
password, ok = dsService.DecryptedValue(&ds, "password")
require.True(t, ok)
require.Empty(t, password)
})
}
func TestService_HTTPClientOptions(t *testing.T) {
cfg := &setting.Cfg{
Azure: &azsettings.AzureSettings{},
@@ -678,10 +571,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
"azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5",
})
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
opts, err := dsService.httpClientOptions(&ds)
opts, err := dsService.httpClientOptions(context.Background(), &ds)
require.NoError(t, err)
require.NotNil(t, opts.Middlewares)
@@ -695,10 +589,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
"httpMethod": "POST",
})
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
opts, err := dsService.httpClientOptions(&ds)
opts, err := dsService.httpClientOptions(context.Background(), &ds)
require.NoError(t, err)
if opts.Middlewares != nil {
@@ -714,10 +609,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
"azureCredentials": "invalid",
})
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
_, err := dsService.httpClientOptions(&ds)
_, err := dsService.httpClientOptions(context.Background(), &ds)
assert.Error(t, err)
})
@@ -732,10 +628,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
"azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5",
})
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
opts, err := dsService.httpClientOptions(&ds)
opts, err := dsService.httpClientOptions(context.Background(), &ds)
require.NoError(t, err)
require.NotNil(t, opts.Middlewares)
@@ -750,10 +647,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
"azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5",
})
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
opts, err := dsService.httpClientOptions(&ds)
opts, err := dsService.httpClientOptions(context.Background(), &ds)
require.NoError(t, err)
if opts.Middlewares != nil {
@@ -772,10 +670,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
"azureEndpointResourceId": "invalid",
})
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
_, err := dsService.httpClientOptions(&ds)
_, err := dsService.httpClientOptions(context.Background(), &ds)
assert.Error(t, err)
})
})
@@ -792,10 +691,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
"azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5",
})
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
opts, err := dsService.httpClientOptions(&ds)
opts, err := dsService.httpClientOptions(context.Background(), &ds)
require.NoError(t, err)
if opts.Middlewares != nil {

11
pkg/services/ngalert/api/api_testing_test.go
View File

@@ -13,6 +13,7 @@ import (
"github.com/grafana/grafana/pkg/services/accesscontrol"
acMock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
"github.com/grafana/grafana/pkg/services/datasources"
fakes "github.com/grafana/grafana/pkg/services/datasources/fakes"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/models"
@@ -61,7 +62,7 @@ func TestRouteTestGrafanaRuleConfig(t *testing.T) {
{Action: datasources.ActionQuery, Scope: datasources.ScopeProvider.GetResourceScopeUID(data2.DatasourceUID)},
})
ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{
ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{
{Uid: data1.DatasourceUID},
{Uid: data2.DatasourceUID},
}}
@@ -102,7 +103,7 @@ func TestRouteTestGrafanaRuleConfig(t *testing.T) {
t.Run("should require user to be signed in", func(t *testing.T) {
data1 := models.GenerateAlertQuery()
ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{
ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{
{Uid: data1.DatasourceUID},
}}
@@ -182,7 +183,7 @@ func TestRouteEvalQueries(t *testing.T) {
{Action: datasources.ActionQuery, Scope: datasources.ScopeProvider.GetResourceScopeUID(data2.DatasourceUID)},
})
ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{
ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{
{Uid: data1.DatasourceUID},
{Uid: data2.DatasourceUID},
}}
@@ -226,7 +227,7 @@ func TestRouteEvalQueries(t *testing.T) {
t.Run("should require user to be signed in", func(t *testing.T) {
data1 := models.GenerateAlertQuery()
ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{
ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{
{Uid: data1.DatasourceUID},
}}
@@ -265,7 +266,7 @@ func TestRouteEvalQueries(t *testing.T) {
})
}
func createTestingApiSrv(ds *datasources.FakeCacheService, ac *acMock.Mock, evaluator *eval.FakeEvaluator) *TestingApiSrv {
func createTestingApiSrv(ds *fakes.FakeCacheService, ac *acMock.Mock, evaluator *eval.FakeEvaluator) *TestingApiSrv {
if ac == nil {
ac = acMock.New().WithDisabled()
}

13
pkg/services/query/query.go
View File

@@ -15,7 +15,6 @@ import (
"github.com/grafana/grafana/pkg/plugins/adapters"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/oauthtoken"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/grafanads"
"github.com/grafana/grafana/pkg/tsdb/legacydata"
@@ -33,7 +32,7 @@ func ProvideService(
dataSourceCache datasources.CacheService,
expressionService *expr.Service,
pluginRequestValidator models.PluginRequestValidator,
SecretsService secrets.Service,
dataSourceService datasources.DataSourceService,
pluginClient plugins.Client,
oAuthTokenService oauthtoken.OAuthTokenService,
) *Service {
@@ -42,7 +41,7 @@ func ProvideService(
dataSourceCache: dataSourceCache,
expressionService: expressionService,
pluginRequestValidator: pluginRequestValidator,
secretsService: SecretsService,
dataSourceService: dataSourceService,
pluginClient: pluginClient,
oAuthTokenService: oAuthTokenService,
log: log.New("query_data"),
@@ -56,7 +55,7 @@ type Service struct {
dataSourceCache datasources.CacheService
expressionService *expr.Service
pluginRequestValidator models.PluginRequestValidator
secretsService secrets.Service
dataSourceService datasources.DataSourceService
pluginClient plugins.Client
oAuthTokenService oauthtoken.OAuthTokenService
log log.Logger
@@ -291,9 +290,9 @@ func (s *Service) getDataSourceFromQuery(ctx context.Context, user *models.Signe
return nil, NewErrBadQuery("missing data source ID/UID")
}
func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(map[string][]byte) map[string]string {
return func(m map[string][]byte) map[string]string {
decryptedJsonData, err := s.secretsService.DecryptJsonData(ctx, m)
func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string {
return func(ds *models.DataSource) map[string]string {
decryptedJsonData, err := s.dataSourceService.DecryptedValues(ctx, ds)
if err != nil {
s.log.Error("Failed to decrypt secure json data", "error", err)
}

45
pkg/services/query/query_test.go
View File

@@ -2,6 +2,7 @@ package query_test
import (
"context"
"encoding/json"
"net/http"
"testing"
@@ -12,18 +13,29 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
datasources "github.com/grafana/grafana/pkg/services/datasources/service"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/query"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/secrets/fakes"
"github.com/grafana/grafana/pkg/services/secrets/kvstore"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/stretchr/testify/require"
)
func TestQueryData(t *testing.T) {
t.Run("it attaches custom headers to the request", func(t *testing.T) {
tc := setup()
tc := setup(t)
tc.dataSourceCache.ds.JsonData = simplejson.NewFromAny(map[string]interface{}{"httpHeaderName1": "foo", "httpHeaderName2": "bar"})
tc.secretService.decryptedJson = map[string]string{"httpHeaderValue1": "test-header", "httpHeaderValue2": "test-header2"}
_, err := tc.queryService.QueryData(context.Background(), nil, true, metricRequest(), false)
secureJsonData, err := json.Marshal(map[string]string{"httpHeaderValue1": "test-header", "httpHeaderValue2": "test-header2"})
require.NoError(t, err)
err = tc.secretStore.Set(context.Background(), tc.dataSourceCache.ds.OrgId, tc.dataSourceCache.ds.Name, "datasource", string(secureJsonData))
require.NoError(t, err)
_, err = tc.queryService.QueryData(context.Background(), nil, true, metricRequest(), false)
require.Nil(t, err)
require.Equal(t, map[string]string{"foo": "test-header", "bar": "test-header2"}, tc.pluginContext.req.Headers)
@@ -36,7 +48,7 @@ func TestQueryData(t *testing.T) {
}
token = token.WithExtra(map[string]interface{}{"id_token": "id-token"})
tc := setup()
tc := setup(t)
tc.oauthTokenService.passThruEnabled = true
tc.oauthTokenService.token = token
@@ -51,26 +63,29 @@ func TestQueryData(t *testing.T) {
})
}
func setup() *testContext {
func setup(t *testing.T) *testContext {
pc := &fakePluginClient{}
sc := &fakeSecretsService{}
dc := &fakeDataSourceCache{ds: &models.DataSource{}}
tc := &fakeOAuthTokenService{}
rv := &fakePluginRequestValidator{}
ss := kvstore.SetupTestService(t)
ssvc := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
ds := datasources.ProvideService(nil, ssvc, ss, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
return &testContext{
pluginContext: pc,
secretService: sc,
secretStore: ss,
dataSourceCache: dc,
oauthTokenService: tc,
pluginRequestValidator: rv,
queryService: query.ProvideService(nil, dc, nil, rv, sc, pc, tc),
queryService: query.ProvideService(nil, dc, nil, rv, ds, pc, tc),
}
}
type testContext struct {
pluginContext *fakePluginClient
secretService *fakeSecretsService
secretStore kvstore.SecretsKVStore
dataSourceCache *fakeDataSourceCache
oauthTokenService *fakeOAuthTokenService
pluginRequestValidator *fakePluginRequestValidator
@@ -108,16 +123,6 @@ func (ts *fakeOAuthTokenService) IsOAuthPassThruEnabled(*models.DataSource) bool
return ts.passThruEnabled
}
type fakeSecretsService struct {
secrets.Service
decryptedJson map[string]string
}
func (s *fakeSecretsService) DecryptJsonData(ctx context.Context, sjd map[string][]byte) (map[string]string, error) {
return s.decryptedJson, nil
}
type fakeDataSourceCache struct {
ds *models.DataSource
}

29
pkg/services/secrets/kvstore/helpers.go Normal file
View File

@@ -0,0 +1,29 @@
package kvstore
import (
"testing"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/secrets/database"
"github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
func SetupTestService(t *testing.T) SecretsKVStore {
t.Helper()
sqlStore := sqlstore.InitTestDB(t)
store := database.ProvideSecretsStore(sqlstore.InitTestDB(t))
secretsService := manager.SetupTestService(t, store)
kv := &secretsKVStoreSQL{
sqlStore: sqlStore,
log: log.New("secrets.kvstore"),
secretsService: secretsService,
decryptionCache: decryptionCache{
cache: make(map[int64]cachedDecrypted),
},
}
return kv
}

77
pkg/services/secrets/kvstore/kvstore.go Normal file
View File

@@ -0,0 +1,77 @@
package kvstore
import (
"context"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
const (
// Wildcard to query all organizations
AllOrganizations = -1
)
func ProvideService(sqlStore sqlstore.Store, secretsService secrets.Service) SecretsKVStore {
return &secretsKVStoreSQL{
sqlStore: sqlStore,
secretsService: secretsService,
log: log.New("secrets.kvstore"),
decryptionCache: decryptionCache{
cache: make(map[int64]cachedDecrypted),
},
}
}
// SecretsKVStore is an interface for k/v store.
type SecretsKVStore interface {
Get(ctx context.Context, orgId int64, namespace string, typ string) (string, bool, error)
Set(ctx context.Context, orgId int64, namespace string, typ string, value string) error
Del(ctx context.Context, orgId int64, namespace string, typ string) error
Keys(ctx context.Context, orgId int64, namespace string, typ string) ([]Key, error)
Rename(ctx context.Context, orgId int64, namespace string, typ string, newNamespace string) error
}
// WithType returns a kvstore wrapper with fixed orgId and type.
func With(kv SecretsKVStore, orgId int64, namespace string, typ string) *FixedKVStore {
return &FixedKVStore{
kvStore: kv,
OrgId: orgId,
Namespace: namespace,
Type: typ,
}
}
// FixedKVStore is a SecretsKVStore wrapper with fixed orgId, namespace and type.
type FixedKVStore struct {
kvStore SecretsKVStore
OrgId int64
Namespace string
Type string
}
func (kv *FixedKVStore) Get(ctx context.Context) (string, bool, error) {
return kv.kvStore.Get(ctx, kv.OrgId, kv.Namespace, kv.Type)
}
func (kv *FixedKVStore) Set(ctx context.Context, value string) error {
return kv.kvStore.Set(ctx, kv.OrgId, kv.Namespace, kv.Type, value)
}
func (kv *FixedKVStore) Del(ctx context.Context) error {
return kv.kvStore.Del(ctx, kv.OrgId, kv.Namespace, kv.Type)
}
func (kv *FixedKVStore) Keys(ctx context.Context) ([]Key, error) {
return kv.kvStore.Keys(ctx, kv.OrgId, kv.Namespace, kv.Type)
}
func (kv *FixedKVStore) Rename(ctx context.Context, newNamespace string) error {
err := kv.kvStore.Rename(ctx, kv.OrgId, kv.Namespace, kv.Type, newNamespace)
if err != nil {
return err
}
kv.Namespace = newNamespace
return nil
}

226
pkg/services/secrets/kvstore/kvstore_test.go Normal file
View File

@@ -0,0 +1,226 @@
package kvstore
import (
"context"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type TestCase struct {
OrgId int64
Namespace string
Type string
Revision int64
}
func (t *TestCase) Value() string {
return fmt.Sprintf("%d:%s:%s:%d", t.OrgId, t.Namespace, t.Type, t.Revision)
}
func TestKVStore(t *testing.T) {
kv := SetupTestService(t)
ctx := context.Background()
testCases := []*TestCase{
{
OrgId: 0,
Namespace: "namespace1",
Type: "testing1",
},
{
OrgId: 0,
Namespace: "namespace2",
Type: "testing2",
},
{
OrgId: 1,
Namespace: "namespace1",
Type: "testing1",
},
{
OrgId: 1,
Namespace: "namespace3",
Type: "testing3",
},
}
for _, tc := range testCases {
err := kv.Set(ctx, tc.OrgId, tc.Namespace, tc.Type, tc.Value())
require.NoError(t, err)
}
t.Run("get existing keys", func(t *testing.T) {
for _, tc := range testCases {
value, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type)
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, tc.Value(), value)
}
})
t.Run("get nonexistent keys", func(t *testing.T) {
tcs := []*TestCase{
{
OrgId: 0,
Namespace: "namespace3",
Type: "testing3",
},
{
OrgId: 1,
Namespace: "namespace2",
Type: "testing2",
},
{
OrgId: 2,
Namespace: "namespace1",
Type: "testing1",
},
}
for _, tc := range tcs {
value, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type)
require.Nil(t, err)
require.False(t, ok)
require.Equal(t, "", value)
}
})
t.Run("modify existing key", func(t *testing.T) {
tc := testCases[0]
value, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type)
require.NoError(t, err)
require.True(t, ok)
assert.Equal(t, tc.Value(), value)
tc.Revision += 1
err = kv.Set(ctx, tc.OrgId, tc.Namespace, tc.Type, tc.Value())
require.NoError(t, err)
value, ok, err = kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type)
require.NoError(t, err)
require.True(t, ok)
assert.Equal(t, tc.Value(), value)
})
t.Run("use fixed client", func(t *testing.T) {
tc := testCases[0]
client := With(kv, tc.OrgId, tc.Namespace, tc.Type)
fmt.Println(client.Namespace, client.OrgId, client.Type)
value, ok, err := client.Get(ctx)
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, tc.Value(), value)
tc.Revision += 1
err = client.Set(ctx, tc.Value())
require.NoError(t, err)
value, ok, err = client.Get(ctx)
require.NoError(t, err)
require.True(t, ok)
assert.Equal(t, tc.Value(), value)
})
t.Run("deleting keys", func(t *testing.T) {
var stillHasKeys bool
for _, tc := range testCases {
if _, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type); err == nil && ok {
stillHasKeys = true
break
}
}
require.True(t, stillHasKeys,
"we are going to test key deletion, but there are no keys to delete in the database")
for _, tc := range testCases {
err := kv.Del(ctx, tc.OrgId, tc.Namespace, tc.Type)
require.NoError(t, err)
}
for _, tc := range testCases {
_, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type)
require.NoError(t, err)
require.False(t, ok, "all keys should be deleted at this point")
}
})
t.Run("listing existing keys", func(t *testing.T) {
kv := SetupTestService(t)
ctx := context.Background()
namespace, typ := "listtest", "listtest"
testCases := []*TestCase{
{
OrgId: 1,
Type: typ,
Namespace: namespace,
},
{
OrgId: 2,
Type: typ,
Namespace: namespace,
},
{
OrgId: 3,
Type: typ,
Namespace: namespace,
},
{
OrgId: 4,
Type: typ,
Namespace: namespace,
},
{
OrgId: 1,
Type: typ,
Namespace: "other_key",
},
{
OrgId: 4,
Type: typ,
Namespace: "another_one",
},
}
for _, tc := range testCases {
err := kv.Set(ctx, tc.OrgId, tc.Namespace, tc.Type, tc.Value())
require.NoError(t, err)
}
keys, err := kv.Keys(ctx, AllOrganizations, namespace, typ)
require.NoError(t, err)
require.Len(t, keys, 4)
found := 0
for _, key := range keys {
for _, tc := range testCases {
if key.OrgId == tc.OrgId && key.Namespace == tc.Namespace && key.Type == tc.Type {
found++
break
}
}
}
require.Equal(t, 4, found, "querying for all orgs should return 4 records")
keys, err = kv.Keys(ctx, 1, namespace, typ)
require.NoError(t, err)
require.Len(t, keys, 1, "querying for a specific org should return 1 record")
keys, err = kv.Keys(ctx, AllOrganizations, "not_existing_namespace", "not_existing_type")
require.NoError(t, err, "querying a not existing namespace should not throw an error")
require.Len(t, keys, 0, "querying a not existing namespace should return an empty slice")
})
}

31
pkg/services/secrets/kvstore/model.go Normal file
View File

@@ -0,0 +1,31 @@
package kvstore
import (
"time"
)
// Item stored in k/v store.
type Item struct {
Id int64
OrgId *int64
Namespace *string
Type *string
Value string
Created time.Time
Updated time.Time
}
func (i *Item) TableName() string {
return "secrets"
}
type Key struct {
OrgId int64
Namespace string
Type string
}
func (i *Key) TableName() string {
return "secrets"
}

220
pkg/services/secrets/kvstore/sql.go Normal file
View File

@@ -0,0 +1,220 @@
package kvstore
import (
"context"
"encoding/base64"
"sync"
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
// secretsKVStoreSQL provides a key/value store backed by the Grafana database
type secretsKVStoreSQL struct {
log log.Logger
sqlStore sqlstore.Store
secretsService secrets.Service
decryptionCache decryptionCache
}
type decryptionCache struct {
cache map[int64]cachedDecrypted
sync.Mutex
}
type cachedDecrypted struct {
updated time.Time
value string
}
var b64 = base64.RawStdEncoding
// Get an item from the store
func (kv *secretsKVStoreSQL) Get(ctx context.Context, orgId int64, namespace string, typ string) (string, bool, error) {
item := Item{
OrgId: &orgId,
Namespace: &namespace,
Type: &typ,
}
var isFound bool
var decryptedValue []byte
err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
has, err := dbSession.Get(&item)
if err != nil {
kv.log.Debug("error getting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
return err
}
if !has {
kv.log.Debug("secret value not found", "orgId", orgId, "type", typ, "namespace", namespace)
return nil
}
isFound = true
kv.log.Debug("got secret value", "orgId", orgId, "type", typ, "namespace", namespace)
return nil
})
if err == nil && isFound {
kv.decryptionCache.Lock()
defer kv.decryptionCache.Unlock()
if cache, present := kv.decryptionCache.cache[item.Id]; present && item.Updated.Equal(cache.updated) {
return cache.value, isFound, err
}
decodedValue, err := b64.DecodeString(item.Value)
if err != nil {
kv.log.Debug("error decoding secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
return string(decryptedValue), isFound, err
}
decryptedValue, err = kv.secretsService.Decrypt(ctx, decodedValue)
if err != nil {
kv.log.Debug("error decrypting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
return string(decryptedValue), isFound, err
}
kv.decryptionCache.cache[item.Id] = cachedDecrypted{
updated: item.Updated,
value: string(decryptedValue),
}
}
return string(decryptedValue), isFound, err
}
// Set an item in the store
func (kv *secretsKVStoreSQL) Set(ctx context.Context, orgId int64, namespace string, typ string, value string) error {
encryptedValue, err := kv.secretsService.Encrypt(ctx, []byte(value), secrets.WithoutScope())
if err != nil {
kv.log.Debug("error encrypting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
return err
}
encodedValue := b64.EncodeToString(encryptedValue)
return kv.sqlStore.WithTransactionalDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
item := Item{
OrgId: &orgId,
Namespace: &namespace,
Type: &typ,
}
has, err := dbSession.Get(&item)
if err != nil {
kv.log.Debug("error checking secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
return err
}
if has && item.Value == encodedValue {
kv.log.Debug("secret value not changed", "orgId", orgId, "type", typ, "namespace", namespace)
return nil
}
item.Value = encodedValue
item.Updated = time.Now()
if has {
// if item already exists we update it
_, err = dbSession.ID(item.Id).Update(&item)
if err != nil {
kv.log.Debug("error updating secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
} else {
kv.decryptionCache.cache[item.Id] = cachedDecrypted{
updated: item.Updated,
value: value,
}
kv.log.Debug("secret value updated", "orgId", orgId, "type", typ, "namespace", namespace)
}
return err
}
// if item doesn't exist we create it
item.Created = item.Updated
_, err = dbSession.Insert(&item)
if err != nil {
kv.log.Debug("error inserting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
} else {
kv.log.Debug("secret value inserted", "orgId", orgId, "type", typ, "namespace", namespace)
}
return err
})
}
// Del deletes an item from the store.
func (kv *secretsKVStoreSQL) Del(ctx context.Context, orgId int64, namespace string, typ string) error {
err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
item := Item{
OrgId: &orgId,
Namespace: &namespace,
Type: &typ,
}
has, err := dbSession.Get(&item)
if err != nil {
kv.log.Debug("error checking secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
return err
}
if has {
// if item exists we delete it
_, err = dbSession.ID(item.Id).Delete(&item)
if err != nil {
kv.log.Debug("error deleting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
} else {
delete(kv.decryptionCache.cache, item.Id)
kv.log.Debug("secret value deleted", "orgId", orgId, "type", typ, "namespace", namespace)
}
return err
}
return nil
})
return err
}
// Keys get all keys for a given namespace. To query for all
// organizations the constant 'kvstore.AllOrganizations' can be passed as orgId.
func (kv *secretsKVStoreSQL) Keys(ctx context.Context, orgId int64, namespace string, typ string) ([]Key, error) {
var keys []Key
err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
query := dbSession.Where("namespace = ?", namespace).And("type = ?", typ)
if orgId != AllOrganizations {
query.And("org_id = ?", orgId)
}
return query.Find(&keys)
})
return keys, err
}
// Rename an item in the store
func (kv *secretsKVStoreSQL) Rename(ctx context.Context, orgId int64, namespace string, typ string, newNamespace string) error {
return kv.sqlStore.WithTransactionalDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
item := Item{
OrgId: &orgId,
Namespace: &namespace,
Type: &typ,
}
has, err := dbSession.Get(&item)
if err != nil {
kv.log.Debug("error checking secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
return err
}
item.Namespace = &newNamespace
item.Updated = time.Now()
if has {
// if item already exists we update it
_, err = dbSession.ID(item.Id).Update(&item)
if err != nil {
kv.log.Debug("error updating secret namespace", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
} else {
kv.log.Debug("secret namespace updated", "orgId", orgId, "type", typ, "namespace", namespace)
}
return err
}
return err
})
}

20
pkg/services/sqlstore/migrations/secrets_mig.go
View File

@@ -18,4 +18,24 @@ func addSecretsMigration(mg *migrator.Migrator) {
}
mg.AddMigration("create data_keys table", migrator.NewAddTableMigration(dataKeysV1))
secretsV1 := migrator.Table{
Name: "secrets",
Columns: []*migrator.Column{
{Name: "id", Type: migrator.DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
{Name: "org_id", Type: migrator.DB_BigInt, Nullable: false},
{Name: "namespace", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
{Name: "type", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
{Name: "value", Type: migrator.DB_Text, Nullable: true},
{Name: "created", Type: migrator.DB_DateTime, Nullable: false},
{Name: "updated", Type: migrator.DB_DateTime, Nullable: false},
},
Indices: []*migrator.Index{
{Cols: []string{"org_id"}},
{Cols: []string{"org_id", "namespace"}},
{Cols: []string{"org_id", "namespace", "type"}, Type: migrator.UniqueIndex},
},
}
mg.AddMigration("create secrets table", migrator.NewAddTableMigration(secretsV1))
}

7
pkg/tsdb/legacydata/service/service.go
View File

@@ -40,6 +40,11 @@ func (h *Service) HandleRequest(ctx context.Context, ds *models.DataSource, quer
return legacydata.DataResponse{}, err
}
decryptedValues, err := h.dataSourcesService.DecryptedValues(ctx, ds)
if err != nil {
return legacydata.DataResponse{}, err
}
instanceSettings := &backend.DataSourceInstanceSettings{
ID: ds.Id,
Name: ds.Name,
@@ -49,7 +54,7 @@ func (h *Service) HandleRequest(ctx context.Context, ds *models.DataSource, quer
BasicAuthEnabled: ds.BasicAuth,
BasicAuthUser: ds.BasicAuthUser,
JSONData: jsonDataBytes,
DecryptedSecureJSONData: h.dataSourcesService.DecryptedValues(ds),
DecryptedSecureJSONData: decryptedValues,
Updated: ds.Updated,
UID: ds.Uid,
}

4
pkg/tsdb/legacydata/service/service_test.go
View File

@@ -13,6 +13,7 @@ import (
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/oauthtoken"
"github.com/grafana/grafana/pkg/services/secrets/fakes"
"github.com/grafana/grafana/pkg/services/secrets/kvstore"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/legacydata"
@@ -38,8 +39,9 @@ func TestHandleRequest(t *testing.T) {
actualReq = req
return backend.NewQueryDataResponse(), nil
}
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
s := ProvideService(client, nil, dsService)
ds := &models.DataSource{Id: 12, Type: "unregisteredType", JsonData: simplejson.New()}