Chore: Refactor secrets service (#40331)

pkg/server/backgroundsvcs/background_services.go

@@ -49,7 +49,7 @@ func ProvideBackgroundServiceRegistry(
 	// Need to make sure these are initialized, is there a better place to put them?
 	_ *azuremonitor.Service, _ *cloudwatch.CloudWatchService, _ *elasticsearch.Service, _ *graphite.Service,
 	_ *influxdb.Service, _ *loki.Service, _ *opentsdb.Service, _ *prometheus.Service, _ *tempo.Service,
-	_ *testdatasource.TestDataPlugin, _ *plugindashboards.Service, _ *dashboardsnapshots.Service, _ secrets.SecretsService,
+	_ *testdatasource.TestDataPlugin, _ *plugindashboards.Service, _ *dashboardsnapshots.Service, _ secrets.Service,
 	_ *postgres.Service, _ *mysql.Service, _ *mssql.Service, _ *grafanads.Service, _ *cloudmonitoring.Service,
 	_ *pluginsettings.Service, _ *alerting.AlertNotificationService,
 ) *BackgroundServiceRegistry {

pkg/server/wire.go

@@ -52,6 +52,8 @@ import (
 	"github.com/grafana/grafana/pkg/services/schemaloader"
 	"github.com/grafana/grafana/pkg/services/search"
 	"github.com/grafana/grafana/pkg/services/secrets"
+	secretsDatabase "github.com/grafana/grafana/pkg/services/secrets/database"
+	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
 	"github.com/grafana/grafana/pkg/services/shorturls"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/setting"
@@ -145,7 +147,10 @@ var wireBasicSet = wire.NewSet(
 	graphite.ProvideService,
 	prometheus.ProvideService,
 	elasticsearch.ProvideService,
-	secrets.ProvideSecretsService,
+	secretsManager.ProvideSecretsService,
+	wire.Bind(new(secrets.Service), new(*secretsManager.SecretsService)),
+	secretsDatabase.ProvideSecretsStore,
+	wire.Bind(new(secrets.Store), new(*secretsDatabase.SecretsStoreImpl)),
 	grafanads.ProvideService,
 	dashboardsnapshots.ProvideService,
 	datasources.ProvideService,

pkg/services/secrets/database.go

@@ -1,78 +0,0 @@
-package secrets
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/grafana/grafana/pkg/infra/log"
-	"github.com/grafana/grafana/pkg/services/secrets/types"
-	"github.com/grafana/grafana/pkg/services/sqlstore"
-)
-
-const dataKeysTable = "data_keys"
-
-var logger = log.New("secrets-store")
-
-func (s *SecretsService) GetDataKey(ctx context.Context, name string) (*types.DataKey, error) {
-	dataKey := &types.DataKey{}
-	var exists bool
-
-	err := s.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
-		var err error
-		exists, err = sess.Table(dataKeysTable).
-			Where("name = ? AND active = ?", name, s.sqlStore.Dialect.BooleanStr(true)).
-			Get(dataKey)
-		return err
-	})
-
-	if !exists {
-		return nil, types.ErrDataKeyNotFound
-	}
-
-	if err != nil {
-		logger.Error("Failed getting data key", "err", err, "name", name)
-		return nil, fmt.Errorf("failed getting data key: %w", err)
-	}
-
-	return dataKey, nil
-}
-
-func (s *SecretsService) GetAllDataKeys(ctx context.Context) ([]*types.DataKey, error) {
-	result := make([]*types.DataKey, 0)
-	err := s.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
-		err := sess.Table(dataKeysTable).Find(&result)
-		return err
-	})
-	return result, err
-}
-
-func (s *SecretsService) CreateDataKey(ctx context.Context, dataKey types.DataKey) error {
-	return s.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
-		return s.CreateDataKeyWithDBSession(ctx, dataKey, sess)
-	})
-}
-
-func (s *SecretsService) CreateDataKeyWithDBSession(_ context.Context, dataKey types.DataKey, sess *sqlstore.DBSession) error {
-	if !dataKey.Active {
-		return fmt.Errorf("cannot insert deactivated data keys")
-	}
-
-	dataKey.Created = time.Now()
-	dataKey.Updated = dataKey.Created
-
-	_, err := sess.Table(dataKeysTable).Insert(&dataKey)
-	return err
-}
-
-func (s *SecretsService) DeleteDataKey(ctx context.Context, name string) error {
-	if len(name) == 0 {
-		return fmt.Errorf("data key name is missing")
-	}
-
-	return s.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
-		_, err := sess.Table(dataKeysTable).Delete(&types.DataKey{Name: name})
-
-		return err
-	})
-}

pkg/services/secrets/database/database.go

@@ -0,0 +1,91 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/grafana/grafana/pkg/services/secrets"
+
+	"xorm.io/xorm"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+)
+
+const dataKeysTable = "data_keys"
+
+var logger = log.New("secrets-store")
+
+type SecretsStoreImpl struct {
+	sqlStore *sqlstore.SQLStore
+}
+
+func ProvideSecretsStore(sqlStore *sqlstore.SQLStore) *SecretsStoreImpl {
+	return &SecretsStoreImpl{
+		sqlStore: sqlStore,
+	}
+}
+
+func (ss *SecretsStoreImpl) GetDataKey(ctx context.Context, name string) (*secrets.DataKey, error) {
+	dataKey := &secrets.DataKey{}
+	var exists bool
+
+	err := ss.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
+		var err error
+		exists, err = sess.Table(dataKeysTable).
+			Where("name = ? AND active = ?", name, ss.sqlStore.Dialect.BooleanStr(true)).
+			Get(dataKey)
+		return err
+	})
+
+	if !exists {
+		return nil, secrets.ErrDataKeyNotFound
+	}
+
+	if err != nil {
+		logger.Error("Failed getting data key", "err", err, "name", name)
+		return nil, fmt.Errorf("failed getting data key: %w", err)
+	}
+
+	return dataKey, nil
+}
+
+func (ss *SecretsStoreImpl) GetAllDataKeys(ctx context.Context) ([]*secrets.DataKey, error) {
+	result := make([]*secrets.DataKey, 0)
+	err := ss.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
+		err := sess.Table(dataKeysTable).Find(&result)
+		return err
+	})
+	return result, err
+}
+
+func (ss *SecretsStoreImpl) CreateDataKey(ctx context.Context, dataKey secrets.DataKey) error {
+	return ss.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
+		return ss.CreateDataKeyWithDBSession(ctx, dataKey, sess.Session)
+	})
+}
+
+func (ss *SecretsStoreImpl) CreateDataKeyWithDBSession(_ context.Context, dataKey secrets.DataKey, sess *xorm.Session) error {
+	if !dataKey.Active {
+		return fmt.Errorf("cannot insert deactivated data keys")
+	}
+
+	dataKey.Created = time.Now()
+	dataKey.Updated = dataKey.Created
+
+	_, err := sess.Table(dataKeysTable).Insert(&dataKey)
+	return err
+}
+
+func (ss *SecretsStoreImpl) DeleteDataKey(ctx context.Context, name string) error {
+	if len(name) == 0 {
+		return fmt.Errorf("data key name is missing")
+	}
+
+	return ss.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
+		_, err := sess.Table(dataKeysTable).Delete(&secrets.DataKey{Name: name})
+
+		return err
+	})
+}

pkg/services/secrets/defaultprovider/grafana_provider.go

@@ -1,9 +1,10 @@
-package secrets
+package defaultprovider
 
 import (
 	"context"
 
 	"github.com/grafana/grafana/pkg/services/encryption"
+	"github.com/grafana/grafana/pkg/services/secrets"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
@@ -12,7 +13,7 @@ type grafanaProvider struct {
 	encryption encryption.Service
 }
 
-func newGrafanaProvider(settings setting.Provider, encryption encryption.Service) grafanaProvider {
+func New(settings setting.Provider, encryption encryption.Service) secrets.Provider {
 	return grafanaProvider{
 		settings:   settings,
 		encryption: encryption,

pkg/services/secrets/fakes/fake_service.go

@@ -0,0 +1,41 @@
+package fakes
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/services/secrets"
+)
+
+type FakeSecretsService struct{}
+
+func NewFakeSecretsService() FakeSecretsService {
+	return FakeSecretsService{}
+}
+
+func (f FakeSecretsService) Encrypt(_ context.Context, payload []byte, _ secrets.EncryptionOptions) ([]byte, error) {
+	return payload, nil
+}
+func (f FakeSecretsService) Decrypt(_ context.Context, payload []byte) ([]byte, error) {
+	return payload, nil
+}
+func (f FakeSecretsService) EncryptJsonData(_ context.Context, kv map[string]string, _ secrets.EncryptionOptions) (map[string][]byte, error) {
+	result := make(map[string][]byte, len(kv))
+	for key, value := range kv {
+		result[key] = []byte(value)
+	}
+	return result, nil
+}
+
+func (f FakeSecretsService) DecryptJsonData(_ context.Context, sjd map[string][]byte) (map[string]string, error) {
+	result := make(map[string]string, len(sjd))
+	for key, value := range sjd {
+		result[key] = string(value)
+	}
+	return result, nil
+}
+func (f FakeSecretsService) GetDecryptedValue(_ context.Context, sjd map[string][]byte, key, fallback string) string {
+	if value, ok := sjd[key]; ok {
+		return string(value)
+	}
+	return fallback
+}

pkg/services/secrets/fakes/fake_store.go

@@ -0,0 +1,47 @@
+package fakes
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/services/secrets"
+	"xorm.io/xorm"
+)
+
+type FakeSecretsStore struct {
+	store map[string]*secrets.DataKey
+}
+
+func NewFakeSecretsStore() FakeSecretsStore {
+	return FakeSecretsStore{store: make(map[string]*secrets.DataKey)}
+}
+
+func (f FakeSecretsStore) GetDataKey(_ context.Context, name string) (*secrets.DataKey, error) {
+	key, ok := f.store[name]
+	if !ok {
+		return nil, secrets.ErrDataKeyNotFound
+	}
+	return key, nil
+}
+
+func (f FakeSecretsStore) GetAllDataKeys(_ context.Context) ([]*secrets.DataKey, error) {
+	result := make([]*secrets.DataKey, 0)
+	for _, key := range f.store {
+		result = append(result, key)
+	}
+	return result, nil
+}
+
+func (f FakeSecretsStore) CreateDataKey(_ context.Context, dataKey secrets.DataKey) error {
+	f.store[dataKey.Name] = &dataKey
+	return nil
+}
+
+func (f FakeSecretsStore) CreateDataKeyWithDBSession(ctx context.Context, dataKey secrets.DataKey, sess *xorm.Session) error {
+	f.store[dataKey.Name] = &dataKey
+	return nil
+}
+
+func (f FakeSecretsStore) DeleteDataKey(_ context.Context, name string) error {
+	delete(f.store, name)
+	return nil
+}

pkg/services/secrets/manager/helpers.go

@@ -1,18 +1,29 @@
-package secrets
+package manager
 
 import (
 	"testing"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/services/encryption/ossencryption"
+	"github.com/grafana/grafana/pkg/services/secrets"
+	"github.com/grafana/grafana/pkg/services/secrets/database"
+	"github.com/grafana/grafana/pkg/services/secrets/fakes"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/stretchr/testify/require"
+
 	"gopkg.in/ini.v1"
 )
 
-func SetupTestService(t *testing.T) SecretsService {
-	t.Helper()
+func SetupTestService(tb testing.TB, db *sqlstore.SQLStore) *SecretsService {
+	if db == nil {
+		return setupTestService(tb, fakes.NewFakeSecretsStore())
+	}
+	return setupTestService(tb, database.ProvideSecretsStore(db))
+}
+
+func setupTestService(tb testing.TB, store secrets.Store) *SecretsService {
+	tb.Helper()
 	defaultKey := "SdlklWklckeLS"
 	if len(setting.SecretKey) > 0 {
 		defaultKey = setting.SecretKey
@@ -20,11 +31,11 @@ func SetupTestService(t *testing.T) SecretsService {
 	raw, err := ini.Load([]byte(`
 		[security]
 		secret_key = ` + defaultKey))
-	require.NoError(t, err)
+	require.NoError(tb, err)
 	settings := &setting.OSSImpl{Cfg: &setting.Cfg{Raw: raw}}
 
 	return ProvideSecretsService(
-		sqlstore.InitTestDB(t),
+		store,
 		bus.New(),
 		ossencryption.ProvideService(),
 		settings,

pkg/services/secrets/manager/manager.go

@@ -0,0 +1,244 @@
+package manager
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/services/encryption"
+	"github.com/grafana/grafana/pkg/services/secrets"
+	grafana "github.com/grafana/grafana/pkg/services/secrets/defaultprovider"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+const defaultProvider = "secretKey"
+
+type SecretsService struct {
+	store    secrets.Store
+	bus      bus.Bus
+	enc      encryption.Service
+	settings setting.Provider
+
+	defaultProvider string
+	providers       map[string]secrets.Provider
+	dataKeyCache    map[string]dataKeyCacheItem
+}
+
+func ProvideSecretsService(store secrets.Store, bus bus.Bus, enc encryption.Service, settings setting.Provider) *SecretsService {
+	providers := map[string]secrets.Provider{
+		defaultProvider: grafana.New(settings, enc),
+	}
+
+	s := &SecretsService{
+		store:           store,
+		bus:             bus,
+		enc:             enc,
+		settings:        settings,
+		defaultProvider: defaultProvider,
+		providers:       providers,
+		dataKeyCache:    make(map[string]dataKeyCacheItem),
+	}
+
+	return s
+}
+
+type dataKeyCacheItem struct {
+	expiry  time.Time
+	dataKey []byte
+}
+
+var b64 = base64.RawStdEncoding
+
+func (s *SecretsService) Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error) {
+	scope := opt()
+	keyName := fmt.Sprintf("%s/%s@%s", time.Now().Format("2006-01-02"), scope, s.defaultProvider)
+
+	dataKey, err := s.dataKey(ctx, keyName)
+	if err != nil {
+		if errors.Is(err, secrets.ErrDataKeyNotFound) {
+			dataKey, err = s.newDataKey(ctx, keyName, scope)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	}
+
+	encrypted, err := s.enc.Encrypt(ctx, payload, string(dataKey))
+	if err != nil {
+		return nil, err
+	}
+
+	prefix := make([]byte, b64.EncodedLen(len(keyName))+2)
+	b64.Encode(prefix[1:], []byte(keyName))
+	prefix[0] = '#'
+	prefix[len(prefix)-1] = '#'
+
+	blob := make([]byte, len(prefix)+len(encrypted))
+	copy(blob, prefix)
+	copy(blob[len(prefix):], encrypted)
+
+	return blob, nil
+}
+
+func (s *SecretsService) Decrypt(ctx context.Context, payload []byte) ([]byte, error) {
+	if len(payload) == 0 {
+		return nil, fmt.Errorf("unable to decrypt empty payload")
+	}
+
+	var dataKey []byte
+
+	if payload[0] != '#' {
+		secretKey := s.settings.KeyValue("security", "secret_key").Value()
+		dataKey = []byte(secretKey)
+	} else {
+		payload = payload[1:]
+		endOfKey := bytes.Index(payload, []byte{'#'})
+		if endOfKey == -1 {
+			return nil, fmt.Errorf("could not find valid key in encrypted payload")
+		}
+		b64Key := payload[:endOfKey]
+		payload = payload[endOfKey+1:]
+		key := make([]byte, b64.DecodedLen(len(b64Key)))
+		_, err := b64.Decode(key, b64Key)
+		if err != nil {
+			return nil, err
+		}
+
+		dataKey, err = s.dataKey(ctx, string(key))
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return s.enc.Decrypt(ctx, payload, string(dataKey))
+}
+
+func (s *SecretsService) EncryptJsonData(ctx context.Context, kv map[string]string, opt secrets.EncryptionOptions) (map[string][]byte, error) {
+	encrypted := make(map[string][]byte)
+	for key, value := range kv {
+		encryptedData, err := s.Encrypt(ctx, []byte(value), opt)
+		if err != nil {
+			return nil, err
+		}
+
+		encrypted[key] = encryptedData
+	}
+	return encrypted, nil
+}
+
+func (s *SecretsService) DecryptJsonData(ctx context.Context, sjd map[string][]byte) (map[string]string, error) {
+	decrypted := make(map[string]string)
+	for key, data := range sjd {
+		decryptedData, err := s.Decrypt(ctx, data)
+		if err != nil {
+			return nil, err
+		}
+
+		decrypted[key] = string(decryptedData)
+	}
+	return decrypted, nil
+}
+
+func (s *SecretsService) GetDecryptedValue(ctx context.Context, sjd map[string][]byte, key, fallback string) string {
+	if value, ok := sjd[key]; ok {
+		decryptedData, err := s.Decrypt(ctx, value)
+		if err != nil {
+			return fallback
+		}
+
+		return string(decryptedData)
+	}
+
+	return fallback
+}
+
+func newRandomDataKey() ([]byte, error) {
+	rawDataKey := make([]byte, 16)
+	_, err := rand.Read(rawDataKey)
+	if err != nil {
+		return nil, err
+	}
+	return rawDataKey, nil
+}
+
+// newDataKey creates a new random DEK, caches it and returns its value
+func (s *SecretsService) newDataKey(ctx context.Context, name string, scope string) ([]byte, error) {
+	// 1. Create new DEK
+	dataKey, err := newRandomDataKey()
+	if err != nil {
+		return nil, err
+	}
+	provider, exists := s.providers[s.defaultProvider]
+	if !exists {
+		return nil, fmt.Errorf("could not find encryption provider '%s'", s.defaultProvider)
+	}
+
+	// 2. Encrypt it
+	encrypted, err := provider.Encrypt(ctx, dataKey)
+	if err != nil {
+		return nil, err
+	}
+
+	// 3. Store its encrypted value in db
+	err = s.store.CreateDataKey(ctx, secrets.DataKey{
+		Active:        true, // TODO: right now we never mark a key as deactivated
+		Name:          name,
+		Provider:      s.defaultProvider,
+		EncryptedData: encrypted,
+		Scope:         scope,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// 4. Cache its unencrypted value and return it
+	s.dataKeyCache[name] = dataKeyCacheItem{
+		expiry:  time.Now().Add(15 * time.Minute),
+		dataKey: dataKey,
+	}
+
+	return dataKey, nil
+}
+
+// dataKey looks up DEK in cache or database, and decrypts it
+func (s *SecretsService) dataKey(ctx context.Context, name string) ([]byte, error) {
+	if item, exists := s.dataKeyCache[name]; exists {
+		if item.expiry.Before(time.Now()) && !item.expiry.IsZero() {
+			delete(s.dataKeyCache, name)
+		} else {
+			return item.dataKey, nil
+		}
+	}
+
+	// 1. get encrypted data key from database
+	dataKey, err := s.store.GetDataKey(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+
+	// 2. decrypt data key
+	provider, exists := s.providers[dataKey.Provider]
+	if !exists {
+		return nil, fmt.Errorf("could not find encryption provider '%s'", dataKey.Provider)
+	}
+
+	decrypted, err := provider.Decrypt(ctx, dataKey.EncryptedData)
+	if err != nil {
+		return nil, err
+	}
+
+	// 3. cache data key
+	s.dataKeyCache[name] = dataKeyCacheItem{
+		expiry:  time.Now().Add(15 * time.Minute),
+		dataKey: decrypted,
+	}
+
+	return decrypted, nil
+}

pkg/services/secrets/manager/manager_test.go

@@ -1,57 +1,62 @@
-package secrets
+package manager
 
 import (
 	"context"
 	"testing"
 
-	"github.com/grafana/grafana/pkg/services/secrets/types"
+	"github.com/grafana/grafana/pkg/services/secrets/database"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/services/secrets"
 )
 
 func TestSecrets_EnvelopeEncryption(t *testing.T) {
-	svc := SetupTestService(t)
+	store := database.ProvideSecretsStore(sqlstore.InitTestDB(t))
+	svc := setupTestService(t, store)
 	ctx := context.Background()
 
 	t.Run("encrypting with no entity_id should create DEK", func(t *testing.T) {
 		plaintext := []byte("very secret string")
 
-		encrypted, err := svc.Encrypt(context.Background(), plaintext, WithoutScope())
+		encrypted, err := svc.Encrypt(context.Background(), plaintext, secrets.WithoutScope())
 		require.NoError(t, err)
 
 		decrypted, err := svc.Decrypt(context.Background(), encrypted)
 		require.NoError(t, err)
 		assert.Equal(t, plaintext, decrypted)
 
-		keys, err := svc.GetAllDataKeys(ctx)
+		keys, err := store.GetAllDataKeys(ctx)
 		require.NoError(t, err)
 		assert.Equal(t, len(keys), 1)
 	})
 	t.Run("encrypting another secret with no entity_id should use the same DEK", func(t *testing.T) {
 		plaintext := []byte("another very secret string")
 
-		encrypted, err := svc.Encrypt(context.Background(), plaintext, WithoutScope())
+		encrypted, err := svc.Encrypt(context.Background(), plaintext, secrets.WithoutScope())
 		require.NoError(t, err)
 
 		decrypted, err := svc.Decrypt(context.Background(), encrypted)
 		require.NoError(t, err)
 		assert.Equal(t, plaintext, decrypted)
 
-		keys, err := svc.GetAllDataKeys(ctx)
+		keys, err := store.GetAllDataKeys(ctx)
 		require.NoError(t, err)
 		assert.Equal(t, len(keys), 1)
 	})
 	t.Run("encrypting with entity_id provided should create a new DEK", func(t *testing.T) {
 		plaintext := []byte("some test data")
 
-		encrypted, err := svc.Encrypt(context.Background(), plaintext, WithScope("user:100"))
+		encrypted, err := svc.Encrypt(context.Background(), plaintext, secrets.WithScope("user:100"))
 		require.NoError(t, err)
 
 		decrypted, err := svc.Decrypt(context.Background(), encrypted)
 		require.NoError(t, err)
 		assert.Equal(t, plaintext, decrypted)
 
-		keys, err := svc.GetAllDataKeys(ctx)
+		keys, err := store.GetAllDataKeys(ctx)
 		require.NoError(t, err)
 		assert.Equal(t, len(keys), 2)
 	})
@@ -73,10 +78,10 @@ func TestSecrets_EnvelopeEncryption(t *testing.T) {
 }
 
 func TestSecretsService_DataKeys(t *testing.T) {
-	svc := SetupTestService(t)
+	store := database.ProvideSecretsStore(sqlstore.InitTestDB(t))
 	ctx := context.Background()
 
-	dataKey := types.DataKey{
+	dataKey := secrets.DataKey{
 		Active:        true,
 		Name:          "test1",
 		Provider:      "test",
@@ -84,16 +89,16 @@ func TestSecretsService_DataKeys(t *testing.T) {
 	}
 
 	t.Run("querying for a DEK that does not exist", func(t *testing.T) {
-		res, err := svc.GetDataKey(ctx, dataKey.Name)
-		assert.ErrorIs(t, types.ErrDataKeyNotFound, err)
+		res, err := store.GetDataKey(ctx, dataKey.Name)
+		assert.ErrorIs(t, secrets.ErrDataKeyNotFound, err)
 		assert.Nil(t, res)
 	})
 
 	t.Run("creating an active DEK", func(t *testing.T) {
-		err := svc.CreateDataKey(ctx, dataKey)
+		err := store.CreateDataKey(ctx, dataKey)
 		require.NoError(t, err)
 
-		res, err := svc.GetDataKey(ctx, dataKey.Name)
+		res, err := store.GetDataKey(ctx, dataKey.Name)
 		require.NoError(t, err)
 		assert.Equal(t, dataKey.EncryptedData, res.EncryptedData)
 		assert.Equal(t, dataKey.Provider, res.Provider)
@@ -102,38 +107,38 @@ func TestSecretsService_DataKeys(t *testing.T) {
 	})
 
 	t.Run("creating an inactive DEK", func(t *testing.T) {
-		k := types.DataKey{
+		k := secrets.DataKey{
 			Active:        false,
 			Name:          "test2",
 			Provider:      "test",
 			EncryptedData: []byte{0x62, 0xAF, 0xA1, 0x1A},
 		}
 
-		err := svc.CreateDataKey(ctx, k)
+		err := store.CreateDataKey(ctx, k)
 		require.Error(t, err)
 
-		res, err := svc.GetDataKey(ctx, k.Name)
-		assert.Equal(t, types.ErrDataKeyNotFound, err)
+		res, err := store.GetDataKey(ctx, k.Name)
+		assert.Equal(t, secrets.ErrDataKeyNotFound, err)
 		assert.Nil(t, res)
 	})
 
 	t.Run("deleting DEK when no name provided must fail", func(t *testing.T) {
-		beforeDelete, err := svc.GetAllDataKeys(ctx)
+		beforeDelete, err := store.GetAllDataKeys(ctx)
 		require.NoError(t, err)
-		err = svc.DeleteDataKey(ctx, "")
+		err = store.DeleteDataKey(ctx, "")
 		require.Error(t, err)
 
-		afterDelete, err := svc.GetAllDataKeys(ctx)
+		afterDelete, err := store.GetAllDataKeys(ctx)
 		require.NoError(t, err)
 		assert.Equal(t, beforeDelete, afterDelete)
 	})
 
 	t.Run("deleting a DEK", func(t *testing.T) {
-		err := svc.DeleteDataKey(ctx, dataKey.Name)
+		err := store.DeleteDataKey(ctx, dataKey.Name)
 		require.NoError(t, err)
 
-		res, err := svc.GetDataKey(ctx, dataKey.Name)
-		assert.Equal(t, types.ErrDataKeyNotFound, err)
+		res, err := store.GetDataKey(ctx, dataKey.Name)
+		assert.Equal(t, secrets.ErrDataKeyNotFound, err)
 		assert.Nil(t, res)
 	})
 }

pkg/services/secrets/secrets.go

@@ -1,269 +1,28 @@
 package secrets
 
 import (
-	"bytes"
 	"context"
-	"crypto/rand"
-	"encoding/base64"
-	"errors"
-	"fmt"
-	"time"
 
-	"github.com/grafana/grafana/pkg/services/sqlstore"
-
-	"github.com/grafana/grafana/pkg/services/encryption"
-
-	"github.com/grafana/grafana/pkg/bus"
-	"github.com/grafana/grafana/pkg/services/secrets/types"
-	"github.com/grafana/grafana/pkg/setting"
+	"xorm.io/xorm"
 )
 
-const defaultProvider = "secretKey"
-
-type SecretsService struct {
-	sqlStore *sqlstore.SQLStore
-	bus      bus.Bus
-	enc      encryption.Service
-	settings setting.Provider
-
-	defaultProvider string
-	providers       map[string]Provider
-	dataKeyCache    map[string]dataKeyCacheItem
+type Service interface {
+	Encrypt(ctx context.Context, payload []byte, opt EncryptionOptions) ([]byte, error)
+	Decrypt(ctx context.Context, payload []byte) ([]byte, error)
+	EncryptJsonData(ctx context.Context, kv map[string]string, opt EncryptionOptions) (map[string][]byte, error)
+	DecryptJsonData(ctx context.Context, sjd map[string][]byte) (map[string]string, error)
+	GetDecryptedValue(ctx context.Context, sjd map[string][]byte, key, fallback string) string
 }
 
-func ProvideSecretsService(sqlStore *sqlstore.SQLStore, bus bus.Bus, enc encryption.Service, settings setting.Provider) SecretsService {
-	providers := map[string]Provider{
-		defaultProvider: newGrafanaProvider(settings, enc),
-	}
-
-	s := SecretsService{
-		sqlStore:        sqlStore,
-		bus:             bus,
-		enc:             enc,
-		settings:        settings,
-		defaultProvider: defaultProvider,
-		providers:       providers,
-		dataKeyCache:    make(map[string]dataKeyCacheItem),
-	}
-
-	return s
-}
-
-type dataKeyCacheItem struct {
-	expiry  time.Time
-	dataKey []byte
+type Store interface {
+	GetDataKey(ctx context.Context, name string) (*DataKey, error)
+	GetAllDataKeys(ctx context.Context) ([]*DataKey, error)
+	CreateDataKey(ctx context.Context, dataKey DataKey) error
+	CreateDataKeyWithDBSession(ctx context.Context, dataKey DataKey, sess *xorm.Session) error
+	DeleteDataKey(ctx context.Context, name string) error
 }
 
 type Provider interface {
 	Encrypt(ctx context.Context, blob []byte) ([]byte, error)
 	Decrypt(ctx context.Context, blob []byte) ([]byte, error)
 }
-
-var b64 = base64.RawStdEncoding
-
-type EncryptionOptions func() string
-
-// WithoutScope uses a root level data key for encryption (DEK),
-// in other words this DEK is not bound to any specific scope (not attached to any user, org, etc.).
-func WithoutScope() EncryptionOptions {
-	return func() string {
-		return "root"
-	}
-}
-
-// WithScope uses a data key for encryption bound to some specific scope (i.e., user, org, etc.).
-// Scope should look like "user:10", "org:1".
-func WithScope(scope string) EncryptionOptions {
-	return func() string {
-		return scope
-	}
-}
-
-func (s *SecretsService) Encrypt(ctx context.Context, payload []byte, opt EncryptionOptions) ([]byte, error) {
-	scope := opt()
-	keyName := fmt.Sprintf("%s/%s@%s", time.Now().Format("2006-01-02"), scope, s.defaultProvider)
-
-	dataKey, err := s.dataKey(ctx, keyName)
-	if err != nil {
-		if errors.Is(err, types.ErrDataKeyNotFound) {
-			dataKey, err = s.newDataKey(ctx, keyName, scope)
-			if err != nil {
-				return nil, err
-			}
-		} else {
-			return nil, err
-		}
-	}
-
-	encrypted, err := s.enc.Encrypt(ctx, payload, string(dataKey))
-	if err != nil {
-		return nil, err
-	}
-
-	prefix := make([]byte, b64.EncodedLen(len(keyName))+2)
-	b64.Encode(prefix[1:], []byte(keyName))
-	prefix[0] = '#'
-	prefix[len(prefix)-1] = '#'
-
-	blob := make([]byte, len(prefix)+len(encrypted))
-	copy(blob, prefix)
-	copy(blob[len(prefix):], encrypted)
-
-	return blob, nil
-}
-
-func (s *SecretsService) Decrypt(ctx context.Context, payload []byte) ([]byte, error) {
-	if len(payload) == 0 {
-		return nil, fmt.Errorf("unable to decrypt empty payload")
-	}
-
-	var dataKey []byte
-
-	if payload[0] != '#' {
-		secretKey := s.settings.KeyValue("security", "secret_key").Value()
-		dataKey = []byte(secretKey)
-	} else {
-		payload = payload[1:]
-		endOfKey := bytes.Index(payload, []byte{'#'})
-		if endOfKey == -1 {
-			return nil, fmt.Errorf("could not find valid key in encrypted payload")
-		}
-		b64Key := payload[:endOfKey]
-		payload = payload[endOfKey+1:]
-		key := make([]byte, b64.DecodedLen(len(b64Key)))
-		_, err := b64.Decode(key, b64Key)
-		if err != nil {
-			return nil, err
-		}
-
-		dataKey, err = s.dataKey(ctx, string(key))
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return s.enc.Decrypt(ctx, payload, string(dataKey))
-}
-
-func (s *SecretsService) EncryptJsonData(ctx context.Context, kv map[string]string, opt EncryptionOptions) (map[string][]byte, error) {
-	encrypted := make(map[string][]byte)
-	for key, value := range kv {
-		encryptedData, err := s.Encrypt(ctx, []byte(value), opt)
-		if err != nil {
-			return nil, err
-		}
-
-		encrypted[key] = encryptedData
-	}
-	return encrypted, nil
-}
-
-func (s *SecretsService) DecryptJsonData(ctx context.Context, sjd map[string][]byte) (map[string]string, error) {
-	decrypted := make(map[string]string)
-	for key, data := range sjd {
-		decryptedData, err := s.Decrypt(ctx, data)
-		if err != nil {
-			return nil, err
-		}
-
-		decrypted[key] = string(decryptedData)
-	}
-	return decrypted, nil
-}
-
-func (s *SecretsService) GetDecryptedValue(ctx context.Context, sjd map[string][]byte, key, fallback string) string {
-	if value, ok := sjd[key]; ok {
-		decryptedData, err := s.Decrypt(ctx, value)
-		if err != nil {
-			return fallback
-		}
-
-		return string(decryptedData)
-	}
-
-	return fallback
-}
-
-func newRandomDataKey() ([]byte, error) {
-	rawDataKey := make([]byte, 16)
-	_, err := rand.Read(rawDataKey)
-	if err != nil {
-		return nil, err
-	}
-	return rawDataKey, nil
-}
-
-// newDataKey creates a new random DEK, caches it and returns its value
-func (s *SecretsService) newDataKey(ctx context.Context, name string, scope string) ([]byte, error) {
-	// 1. Create new DEK
-	dataKey, err := newRandomDataKey()
-	if err != nil {
-		return nil, err
-	}
-	provider, exists := s.providers[s.defaultProvider]
-	if !exists {
-		return nil, fmt.Errorf("could not find encryption provider '%s'", s.defaultProvider)
-	}
-
-	// 2. Encrypt it
-	encrypted, err := provider.Encrypt(ctx, dataKey)
-	if err != nil {
-		return nil, err
-	}
-
-	// 3. Store its encrypted value in db
-	err = s.CreateDataKey(ctx, types.DataKey{
-		Active:        true, // TODO: right now we never mark a key as deactivated
-		Name:          name,
-		Provider:      s.defaultProvider,
-		EncryptedData: encrypted,
-		Scope:         scope,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	// 4. Cache its unencrypted value and return it
-	s.dataKeyCache[name] = dataKeyCacheItem{
-		expiry:  time.Now().Add(15 * time.Minute),
-		dataKey: dataKey,
-	}
-
-	return dataKey, nil
-}
-
-// dataKey looks up DEK in cache or database, and decrypts it
-func (s *SecretsService) dataKey(ctx context.Context, name string) ([]byte, error) {
-	if item, exists := s.dataKeyCache[name]; exists {
-		if item.expiry.Before(time.Now()) && !item.expiry.IsZero() {
-			delete(s.dataKeyCache, name)
-		} else {
-			return item.dataKey, nil
-		}
-	}
-
-	// 1. get encrypted data key from database
-	dataKey, err := s.GetDataKey(ctx, name)
-	if err != nil {
-		return nil, err
-	}
-
-	// 2. decrypt data key
-	provider, exists := s.providers[dataKey.Provider]
-	if !exists {
-		return nil, fmt.Errorf("could not find encryption provider '%s'", dataKey.Provider)
-	}
-
-	decrypted, err := provider.Decrypt(ctx, dataKey.EncryptedData)
-	if err != nil {
-		return nil, err
-	}
-
-	// 3. cache data key
-	s.dataKeyCache[name] = dataKeyCacheItem{
-		expiry:  time.Now().Add(15 * time.Minute),
-		dataKey: decrypted,
-	}
-
-	return decrypted, nil
-}

pkg/services/secrets/types.go

@@ -0,0 +1,36 @@
+package secrets
+
+import (
+	"errors"
+	"time"
+)
+
+var ErrDataKeyNotFound = errors.New("data key not found")
+
+type DataKey struct {
+	Active        bool
+	Name          string
+	Scope         string
+	Provider      string
+	EncryptedData []byte
+	Created       time.Time
+	Updated       time.Time
+}
+
+type EncryptionOptions func() string
+
+// WithoutScope uses a root level data key for encryption (DEK),
+// in other words this DEK is not bound to any specific scope (not attached to any user, org, etc.).
+func WithoutScope() EncryptionOptions {
+	return func() string {
+		return "root"
+	}
+}
+
+// WithScope uses a data key for encryption bound to some specific scope (i.e., user, org, etc.).
+// Scope should look like "user:10", "org:1".
+func WithScope(scope string) EncryptionOptions {
+	return func() string {
+		return scope
+	}
+}

pkg/services/secrets/types/types.go

@@ -1,18 +0,0 @@
-package types
-
-import (
-	"errors"
-	"time"
-)
-
-var ErrDataKeyNotFound = errors.New("data key not found")
-
-type DataKey struct {
-	Active        bool
-	Name          string
-	Scope         string
-	Provider      string
-	EncryptedData []byte
-	Created       time.Time
-	Updated       time.Time
-}