IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Encryption: Cache new DEKs (only) after a caution period (#60664)

Browse Source 
* Encryption: Cache new DEKs (only) after commit

* Fix typo

* Update secrets manager tests with new failing case

* Update secrets manager tests with new clarifications (comments)

* Correct broken method calls

* Unify methods

* Cache data keys only after a caution period

* Caution period for data keys caching only for encrypt ops
This commit is contained in:
Joan López de la Franca Beltran 2023-01-26 10:54:31 +01:00 committed by GitHub
parent 3a442610d2
commit c4e067d49d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 301 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
pkg/services/secrets/database/database.go
View File

@ -5,8 +5,6 @@ import (
"fmt"
"time"
"xorm.io/xorm"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/kmsproviders"
@ -16,22 +14,24 @@ import (
const dataKeysTable = "data_keys"
type SecretsStoreImpl struct {
sqlStore db.DB
log log.Logger
db db.DB
log log.Logger
}
func ProvideSecretsStore(sqlStore db.DB) *SecretsStoreImpl {
return &SecretsStoreImpl{
sqlStore: sqlStore,
log: log.New("secrets.store"),
func ProvideSecretsStore(db db.DB) *SecretsStoreImpl {
store := &SecretsStoreImpl{
db: db,
log: log.New("secrets.store"),
}
return store
}
func (ss *SecretsStoreImpl) GetDataKey(ctx context.Context, id string) (*secrets.DataKey, error) {
dataKey := &secrets.DataKey{}
var exists bool
err := ss.sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
err := ss.db.WithDbSession(ctx, func(sess *db.Session) error {
var err error
exists, err = sess.Table(dataKeysTable).
Where("name = ?", id).
@ -54,10 +54,10 @@ func (ss *SecretsStoreImpl) GetCurrentDataKey(ctx context.Context, label string)
dataKey := &secrets.DataKey{}
var exists bool
err := ss.sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
err := ss.db.WithDbSession(ctx, func(sess *db.Session) error {
var err error
exists, err = sess.Table(dataKeysTable).
Where("label = ? AND active = ?", label, ss.sqlStore.GetDialect().BooleanStr(true)).
Where("label = ? AND active = ?", label, ss.db.GetDialect().BooleanStr(true)).
Get(dataKey)
return err
})
@ -75,7 +75,7 @@ func (ss *SecretsStoreImpl) GetCurrentDataKey(ctx context.Context, label string)
func (ss *SecretsStoreImpl) GetAllDataKeys(ctx context.Context) ([]*secrets.DataKey, error) {
result := make([]*secrets.DataKey, 0)
err := ss.sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
err := ss.db.WithDbSession(ctx, func(sess *db.Session) error {
err := sess.Table(dataKeysTable).Find(&result)
return err
})
@ -83,12 +83,6 @@ func (ss *SecretsStoreImpl) GetAllDataKeys(ctx context.Context) ([]*secrets.Data
}
func (ss *SecretsStoreImpl) CreateDataKey(ctx context.Context, dataKey *secrets.DataKey) error {
return ss.sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
return ss.CreateDataKeyWithDBSession(ctx, dataKey, sess.Session)
})
}
func (ss *SecretsStoreImpl) CreateDataKeyWithDBSession(_ context.Context, dataKey *secrets.DataKey, sess *xorm.Session) error {
if !dataKey.Active {
return fmt.Errorf("cannot insert deactivated data keys")
}
@ -96,14 +90,20 @@ func (ss *SecretsStoreImpl) CreateDataKeyWithDBSession(_ context.Context, dataKe
dataKey.Created = time.Now()
dataKey.Updated = dataKey.Created
_, err := sess.Table(dataKeysTable).Insert(dataKey)
return err
return ss.db.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
_, err := sess.Table(dataKeysTable).Insert(dataKey)
if err != nil {
return err
}
return nil
})
}
func (ss *SecretsStoreImpl) DisableDataKeys(ctx context.Context) error {
return ss.sqlStore.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
return ss.db.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
_, err := sess.Table(dataKeysTable).
Where("active = ?", ss.sqlStore.GetDialect().BooleanStr(true)).
Where("active = ?", ss.db.GetDialect().BooleanStr(true)).
UseBool("active").Update(&secrets.DataKey{Active: false})
return err
})
@ -114,7 +114,7 @@ func (ss *SecretsStoreImpl) DeleteDataKey(ctx context.Context, id string) error
return fmt.Errorf("data key id is missing")
}
return ss.sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
return ss.db.WithDbSession(ctx, func(sess *db.Session) error {
_, err := sess.Table(dataKeysTable).Delete(&secrets.DataKey{Id: id})
return err
@ -127,14 +127,14 @@ func (ss *SecretsStoreImpl) ReEncryptDataKeys(
currProvider secrets.ProviderID,
) error {
keys := make([]*secrets.DataKey, 0)
if err := ss.sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
if err := ss.db.WithDbSession(ctx, func(sess *db.Session) error {
return sess.Table(dataKeysTable).Find(&keys)
}); err != nil {
return err
}
for _, k := range keys {
err := ss.sqlStore.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
err := ss.db.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
provider, ok := providers[kmsproviders.NormalizeProviderID(k.Provider)]
if !ok {
ss.log.Warn(

6
pkg/services/secrets/fakes/fake_store.go
View File

@ -4,7 +4,6 @@ import (
"context"
"github.com/grafana/grafana/pkg/services/secrets"
"xorm.io/xorm"
)
type FakeSecretsStore struct {
@ -47,11 +46,6 @@ func (f FakeSecretsStore) CreateDataKey(_ context.Context, dataKey *secrets.Data
return nil
}
func (f FakeSecretsStore) CreateDataKeyWithDBSession(_ context.Context, dataKey *secrets.DataKey, _ *xorm.Session) error {
f.store[dataKey.Id] = dataKey
return nil
}
func (f FakeSecretsStore) DisableDataKeys(_ context.Context) error {
for id := range f.store {
f.store[id].Active = false

14
pkg/services/secrets/manager/cache.go
View File

@ -8,10 +8,6 @@ import (
"github.com/prometheus/client_golang/prometheus"
)
var (
now = time.Now
)
type dataKeyCacheEntry struct {
id string
label string
@ -75,13 +71,21 @@ func (c *dataKeyCache) getByLabel(label string) (*dataKeyCacheEntry, bool) {
return entry, true
}
func (c *dataKeyCache) add(entry *dataKeyCacheEntry) {
func (c *dataKeyCache) addById(entry *dataKeyCacheEntry) {
c.mtx.Lock()
defer c.mtx.Unlock()
entry.expiration = now().Add(c.cacheTTL)
c.byId[entry.id] = entry
}
func (c *dataKeyCache) addByLabel(entry *dataKeyCacheEntry) {
c.mtx.Lock()
defer c.mtx.Unlock()
entry.expiration = now().Add(c.cacheTTL)
c.byLabel[entry.label] = entry
}

106
pkg/services/secrets/manager/manager.go
View File

@ -19,15 +19,21 @@ import (
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
"github.com/prometheus/client_golang/prometheus"
"golang.org/x/sync/errgroup"
"xorm.io/xorm"
)
const (
keyIdDelimiter = '#'
)
var (
// now is used for testing purposes,
// as a way to fake time.Now function.
now = time.Now
)
type SecretsService struct {
store secrets.Store
enc encryption.Internal
@ -152,10 +158,6 @@ func (s *SecretsService) encryptedWithEnvelopeEncryption(payload []byte) bool {
var b64 = base64.RawStdEncoding
func (s *SecretsService) Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error) {
return s.EncryptWithDBSession(ctx, payload, opt, nil)
}
func (s *SecretsService) EncryptWithDBSession(ctx context.Context, payload []byte, opt secrets.EncryptionOptions, sess *xorm.Session) ([]byte, error) {
// Use legacy encryption service if featuremgmt.FlagDisableEnvelopeEncryption toggle is on
if s.features.IsEnabled(featuremgmt.FlagDisableEnvelopeEncryption) {
return s.enc.Encrypt(ctx, payload, setting.SecretKey)
@ -175,7 +177,7 @@ func (s *SecretsService) EncryptWithDBSession(ctx context.Context, payload []byt
var id string
var dataKey []byte
id, dataKey, err = s.currentDataKey(ctx, label, scope, sess)
id, dataKey, err = s.currentDataKey(ctx, label, scope)
if err != nil {
s.log.Error("Failed to get current data key", "error", err, "label", label)
return nil, err
@ -203,7 +205,7 @@ func (s *SecretsService) EncryptWithDBSession(ctx context.Context, payload []byt
// currentDataKey looks up for current data key in cache or database by name, and decrypts it.
// If there's no current data key in cache nor in database it generates a new random data key,
// and stores it into both the in-memory cache and database (encrypted by the encryption provider).
func (s *SecretsService) currentDataKey(ctx context.Context, label string, scope string, sess *xorm.Session) (string, []byte, error) {
func (s *SecretsService) currentDataKey(ctx context.Context, label string, scope string) (string, []byte, error) {
// We want only one request fetching current data key at time to
// avoid the creation of multiple ones in case there's no one existing.
s.mtx.Lock()
@ -217,7 +219,7 @@ func (s *SecretsService) currentDataKey(ctx context.Context, label string, scope
// If no existing data key was found, create a new one
if dataKey == nil {
id, dataKey, err = s.newDataKey(ctx, label, scope, sess)
id, dataKey, err = s.newDataKey(ctx, label, scope)
if err != nil {
return "", nil, err
}
@ -226,7 +228,7 @@ func (s *SecretsService) currentDataKey(ctx context.Context, label string, scope
return id, dataKey, nil
}
// dataKeyByLabel looks up for data key in cache.
// dataKeyByLabel looks up for data key in cache by label.
// Otherwise, it fetches it from database, decrypts it and caches it decrypted.
func (s *SecretsService) dataKeyByLabel(ctx context.Context, label string) (string, []byte, error) {
// 0. Get data key from in-memory cache.
@ -256,18 +258,13 @@ func (s *SecretsService) dataKeyByLabel(ctx context.Context, label string) (stri
}
// 3. Store the decrypted data key into the in-memory cache.
s.dataKeyCache.add(&dataKeyCacheEntry{
id: dataKey.Id,
label: dataKey.Label,
dataKey: decrypted,
active: dataKey.Active,
})
s.cacheDataKey(dataKey, decrypted)
return dataKey.Id, decrypted, nil
}
// newDataKey creates a new random data key, encrypts it and stores it into the database and cache.
func (s *SecretsService) newDataKey(ctx context.Context, label string, scope string, sess *xorm.Session) (string, []byte, error) {
func (s *SecretsService) newDataKey(ctx context.Context, label string, scope string) (string, []byte, error) {
// 1. Create new data key.
dataKey, err := newRandomDataKey()
if err != nil {
@ -288,6 +285,7 @@ func (s *SecretsService) newDataKey(ctx context.Context, label string, scope str
// 3. Store its encrypted value into the DB.
id := util.GenerateShortUID()
dbDataKey := secrets.DataKey{
Active: true,
Id: id,
@ -297,24 +295,11 @@ func (s *SecretsService) newDataKey(ctx context.Context, label string, scope str
Scope: scope,
}
if sess == nil {
err = s.store.CreateDataKey(ctx, &dbDataKey)
} else {
err = s.store.CreateDataKeyWithDBSession(ctx, &dbDataKey, sess)
}
err = s.store.CreateDataKey(ctx, &dbDataKey)
if err != nil {
return "", nil, err
}
// 4. Store the decrypted data key into the in-memory cache.
s.dataKeyCache.add(&dataKeyCacheEntry{
id: id,
label: label,
dataKey: dataKey,
active: true,
})
return id, dataKey, nil
}
@ -388,13 +373,9 @@ func (s *SecretsService) Decrypt(ctx context.Context, payload []byte) ([]byte, e
}
func (s *SecretsService) EncryptJsonData(ctx context.Context, kv map[string]string, opt secrets.EncryptionOptions) (map[string][]byte, error) {
return s.EncryptJsonDataWithDBSession(ctx, kv, opt, nil)
}
func (s *SecretsService) EncryptJsonDataWithDBSession(ctx context.Context, kv map[string]string, opt secrets.EncryptionOptions, sess *xorm.Session) (map[string][]byte, error) {
encrypted := make(map[string][]byte)
for key, value := range kv {
encryptedData, err := s.EncryptWithDBSession(ctx, []byte(value), opt, sess)
encryptedData, err := s.Encrypt(ctx, []byte(value), opt)
if err != nil {
return nil, err
}
@ -457,12 +438,7 @@ func (s *SecretsService) dataKeyById(ctx context.Context, id string) ([]byte, er
}
// 3. Store the decrypted data key into the in-memory cache.
s.dataKeyCache.add(&dataKeyCacheEntry{
id: dataKey.Id,
label: dataKey.Label,
dataKey: decrypted,
active: dataKey.Active,
})
s.cacheDataKey(dataKey, decrypted)
return decrypted, nil
}
@ -547,3 +523,51 @@ func (s *SecretsService) Run(ctx context.Context) error {
}
}
}
// Caching a data key is tricky, because at SecretsService level we cannot guarantee
// that a newly created data key has actually been persisted, depending on the different
// use cases that rely on SecretsService encryption and different database engines that
// we have support for, because the data key creation may have happened within a DB TX,
// that may fail afterwards.
//
// Therefore, if we cache a data key that hasn't been persisted with success (and won't),
// and later that one is used for a encryption operation (aside from the DB TX that created
// it), we may end up with data encrypted by a non-persisted data key, which could end up
// in (unrecoverable) data corruption.
//
// So, we cache the data key by id and/or by label, depending on the data key's lifetime,
// assuming that a data key older than a "caution period" should have been persisted.
//
// Look at the comments inline for further details.
// You can also take a look at the issue below for more context:
// https://github.com/grafana/grafana-enterprise/issues/4252
func (s *SecretsService) cacheDataKey(dataKey *secrets.DataKey, decrypted []byte) {
// First, we cache the data key by id, because cache "by id" is
// only used by decrypt operations, so no risk of corrupting data.
entry := &dataKeyCacheEntry{
id: dataKey.Id,
label: dataKey.Label,
dataKey: decrypted,
active: dataKey.Active,
}
s.dataKeyCache.addById(entry)
// Then, we cache the data key by label, ONLY if data key's lifetime
// is longer than a certain "caution period", because cache "by label"
// is used (only) by encrypt operations, and we want to ensure that
// no data key is cached for encryption ops before being persisted.
const cautionPeriod = 10 * time.Minute
// We consider a "caution period" of 10m to be long enough for any database
// transaction that implied a data key creation to have finished successfully.
//
// Therefore, we consider that if we fetch a data key from the database,
// more than 10m later than its creation, it should have been actually
// persisted - i.e. the transaction that created it is no longer running.
nowMinusCautionPeriod := now().Add(-cautionPeriod)
if dataKey.Created.Before(nowMinusCautionPeriod) {
s.dataKeyCache.addByLabel(entry)
}
}

183
pkg/services/secrets/manager/manager_test.go
View File

@ -2,6 +2,7 @@ package manager
import (
"context"
"errors"
"testing"
"time"
@ -17,12 +18,14 @@ import (
"github.com/grafana/grafana/pkg/services/kmsproviders/osskmsproviders"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/secrets/database"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
)
func TestSecretsService_EnvelopeEncryption(t *testing.T) {
store := database.ProvideSecretsStore(db.InitTestDB(t))
testDB := db.InitTestDB(t)
store := database.ProvideSecretsStore(testDB)
svc := SetupTestService(t, store)
ctx := context.Background()
@ -82,7 +85,8 @@ func TestSecretsService_EnvelopeEncryption(t *testing.T) {
}
func TestSecretsService_DataKeys(t *testing.T) {
store := database.ProvideSecretsStore(db.InitTestDB(t))
testDB := db.InitTestDB(t)
store := database.ProvideSecretsStore(testDB)
ctx := context.Background()
dataKey := &secrets.DataKey{
@ -160,7 +164,8 @@ func TestSecretsService_DataKeys(t *testing.T) {
func TestSecretsService_UseCurrentProvider(t *testing.T) {
t.Run("When encryption_provider is not specified explicitly, should use 'secretKey' as a current provider", func(t *testing.T) {
svc := SetupTestService(t, database.ProvideSecretsStore(db.InitTestDB(t)))
testDB := db.InitTestDB(t)
svc := SetupTestService(t, database.ProvideSecretsStore(testDB))
assert.Equal(t, secrets.ProviderID("secretKey.v1"), svc.currentProviderID)
})
@ -187,7 +192,8 @@ func TestSecretsService_UseCurrentProvider(t *testing.T) {
features := featuremgmt.WithFeatures()
kms := newFakeKMS(osskmsproviders.ProvideService(encryptionService, settings, features))
secretStore := database.ProvideSecretsStore(db.InitTestDB(t))
testDB := db.InitTestDB(t)
secretStore := database.ProvideSecretsStore(testDB)
secretsService, err := ProvideSecretsService(
secretStore,
@ -261,8 +267,8 @@ func (f *fakeKMS) Provide() (map[secrets.ProviderID]secrets.Provider, error) {
func TestSecretsService_Run(t *testing.T) {
ctx := context.Background()
sql := db.InitTestDB(t)
store := database.ProvideSecretsStore(sql)
testDB := db.InitTestDB(t)
store := database.ProvideSecretsStore(testDB)
svc := SetupTestService(t, store)
t.Run("should stop with no error once the context's finished", func(t *testing.T) {
@ -274,16 +280,26 @@ func TestSecretsService_Run(t *testing.T) {
})
t.Run("should trigger cache clean up", func(t *testing.T) {
// Encrypt to ensure there's a data encryption key generated
_, err := svc.Encrypt(ctx, []byte("grafana"), secrets.WithoutScope())
restoreTimeNowAfterTestExec(t)
// Encrypt to force data encryption key generation
encrypted, err := svc.Encrypt(ctx, []byte("grafana"), secrets.WithoutScope())
require.NoError(t, err)
// Ten minutes later (after caution period)
// Look SecretsService.cacheDataKey for more details.
now = func() time.Time { return time.Now().Add(10 * time.Minute) }
// Decrypt to ensure data encryption key is cached
_, err = svc.Decrypt(ctx, encrypted)
require.NoError(t, err)
// Data encryption key cache should contain one element
require.Len(t, svc.dataKeyCache.byId, 1)
require.Len(t, svc.dataKeyCache.byLabel, 1)
t.Cleanup(func() { now = time.Now })
now = func() time.Time { return time.Now().Add(10 * time.Minute) }
// Twenty minutes later (after caution period + cache ttl)
now = func() time.Time { return time.Now().Add(20 * time.Minute) }
ctx, cancel := context.WithTimeout(ctx, 1*time.Second)
defer cancel()
@ -301,8 +317,8 @@ func TestSecretsService_Run(t *testing.T) {
func TestSecretsService_ReEncryptDataKeys(t *testing.T) {
ctx := context.Background()
sql := db.InitTestDB(t)
store := database.ProvideSecretsStore(sql)
testDB := db.InitTestDB(t)
store := database.ProvideSecretsStore(testDB)
svc := SetupTestService(t, store)
// Encrypt to generate data encryption key
@ -326,6 +342,12 @@ func TestSecretsService_ReEncryptDataKeys(t *testing.T) {
})
t.Run("data keys cache should be invalidated", func(t *testing.T) {
restoreTimeNowAfterTestExec(t)
// Ten minutes later (after caution period)
// Look SecretsService.cacheDataKey for more details.
now = func() time.Time { return time.Now().Add(10 * time.Minute) }
// Decrypt to ensure data key is cached
_, err := svc.Decrypt(ctx, ciphertext)
require.NoError(t, err)
@ -342,7 +364,8 @@ func TestSecretsService_ReEncryptDataKeys(t *testing.T) {
func TestSecretsService_Decrypt(t *testing.T) {
ctx := context.Background()
store := database.ProvideSecretsStore(db.InitTestDB(t))
testDB := db.InitTestDB(t)
store := database.ProvideSecretsStore(testDB)
t.Run("empty payload should fail", func(t *testing.T) {
svc := SetupTestService(t, store)
@ -401,3 +424,137 @@ func TestSecretsService_Decrypt(t *testing.T) {
assert.Equal(t, []byte("grafana"), decrypted)
})
}
func TestIntegration_SecretsService(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
ctx := context.Background()
someData := []byte(`some-data`)
tcs := map[string]func(*testing.T, *sqlstore.SQLStore, *SecretsService){
"regular": func(t *testing.T, _ *sqlstore.SQLStore, svc *SecretsService) {
// We encrypt some data normally, no transactions implied.
_, err := svc.Encrypt(ctx, someData, secrets.WithoutScope())
require.NoError(t, err)
},
"within successful InTransaction": func(t *testing.T, store *sqlstore.SQLStore, svc *SecretsService) {
require.NoError(t, store.InTransaction(ctx, func(ctx context.Context) error {
// We encrypt some data within a transaction that shares the db session.
_, err := svc.Encrypt(ctx, someData, secrets.WithoutScope())
require.NoError(t, err)
// And the transition succeeds.
return nil
}))
},
"within unsuccessful InTransaction": func(t *testing.T, store *sqlstore.SQLStore, svc *SecretsService) {
require.NotNil(t, store.InTransaction(ctx, func(ctx context.Context) error {
// We encrypt some data within a transaction that shares the db session.
_, err := svc.Encrypt(ctx, someData, secrets.WithoutScope())
require.NoError(t, err)
// But the transaction fails.
return errors.New("error")
}))
},
"within unsuccessful InTransaction (plus forced db fetch)": func(t *testing.T, store *sqlstore.SQLStore, svc *SecretsService) {
require.NotNil(t, store.InTransaction(ctx, func(ctx context.Context) error {
// We encrypt some data within a transaction that shares the db session.
encrypted, err := svc.Encrypt(ctx, someData, secrets.WithoutScope())
require.NoError(t, err)
// At this point the data key is not cached yet because
// the transaction haven't been committed yet,
// and won't, so we do a decrypt operation within the
// transaction to force the data key to be
// (potentially) cached (it shouldn't to prevent issues).
decrypted, err := svc.Decrypt(ctx, encrypted)
require.NoError(t, err)
assert.Equal(t, someData, decrypted)
// But the transaction fails.
return errors.New("error")
}))
},
"within successful WithTransactionalDbSession": func(t *testing.T, store *sqlstore.SQLStore, svc *SecretsService) {
require.NoError(t, store.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error {
// We encrypt some data within a transaction that does not share the db session.
_, err := svc.Encrypt(ctx, someData, secrets.WithoutScope())
require.NoError(t, err)
// And the transition succeeds.
return nil
}))
},
"within unsuccessful WithTransactionalDbSession": func(t *testing.T, store *sqlstore.SQLStore, svc *SecretsService) {
require.NotNil(t, store.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error {
// We encrypt some data within a transaction that does not share the db session.
_, err := svc.Encrypt(ctx, someData, secrets.WithoutScope())
require.NoError(t, err)
// But the transaction fails.
return errors.New("error")
}))
},
"within unsuccessful WithTransactionalDbSession (plus forced db fetch)": func(t *testing.T, store *sqlstore.SQLStore, svc *SecretsService) {
require.NotNil(t, store.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error {
// We encrypt some data within a transaction that does not share the db session.
encrypted, err := svc.Encrypt(ctx, someData, secrets.WithoutScope())
require.NoError(t, err)
// At this point the data key is not cached yet because
// the transaction haven't been committed yet,
// and won't, so we do a decrypt operation within the
// transaction to force the data key to be
// (potentially) cached (it shouldn't to prevent issues).
decrypted, err := svc.Decrypt(ctx, encrypted)
require.NoError(t, err)
assert.Equal(t, someData, decrypted)
// But the transaction fails.
return errors.New("error")
}))
},
}
for name, tc := range tcs {
t.Run(name, func(t *testing.T) {
testDB := db.InitTestDB(t)
svc := SetupTestService(t, database.ProvideSecretsStore(testDB))
// Here's what actually matters and varies on each test: look at the test case name.
//
// For historical reasons, and in an old implementation, when a successful encryption
// operation happened within an unsuccessful transaction, the data key was used to be
// cached in memory for the next encryption operations, which caused some data to be
// encrypted with a data key that haven't actually been persisted into the database.
tc(t, testDB, svc)
// Therefore, the data encrypted after this point, become unrecoverable after a restart.
// So, the different test cases here are there to prevent that from happening again
// in the future, whatever it is what happens.
// So, we proceed with an encryption operation:
toEncrypt := []byte(`data-to-encrypt`)
encrypted, err := svc.Encrypt(ctx, toEncrypt, secrets.WithoutScope())
require.NoError(t, err)
// We simulate an instance restart. So, there's no data in the in-memory cache.
svc.dataKeyCache.flush()
// And then, we MUST still be able to decrypt the previously encrypted data:
decrypted, err := svc.Decrypt(ctx, encrypted)
require.NoError(t, err)
assert.Equal(t, toEncrypt, decrypted)
})
}
}
// Use this function at the beginning of those tests
// that manipulates 'now', so it'll leave it in a
// correct state once test execution finishes.
func restoreTimeNowAfterTestExec(t *testing.T) {
t.Helper()
t.Cleanup(func() { now = time.Now })
}

53
pkg/services/secrets/migrator/reencrypt.go
View File

@ -10,6 +10,7 @@ import (
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
func (s simpleSecret) reencrypt(ctx context.Context, secretsSrv *manager.SecretsService, sqlStore db.DB) bool {
@ -32,21 +33,24 @@ func (s simpleSecret) reencrypt(ctx context.Context, secretsSrv *manager.Secrets
continue
}
err := sqlStore.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
err := sqlStore.InTransaction(ctx, func(ctx context.Context) error {
decrypted, err := secretsSrv.Decrypt(ctx, row.Secret)
if err != nil {
logger.Warn("Could not decrypt secret while re-encrypting it", "table", s.tableName, "id", row.Id, "error", err)
return err
}
encrypted, err := secretsSrv.EncryptWithDBSession(ctx, decrypted, secrets.WithoutScope(), sess.Session)
encrypted, err := secretsSrv.Encrypt(ctx, decrypted, secrets.WithoutScope())
if err != nil {
logger.Warn("Could not encrypt secret while re-encrypting it", "table", s.tableName, "id", row.Id, "error", err)
return err
}
updateSQL := fmt.Sprintf("UPDATE %s SET %s = ?, updated = ? WHERE id = ?", s.tableName, s.columnName)
if _, err = sess.Exec(updateSQL, encrypted, nowInUTC(), row.Id); err != nil {
if err = sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
_, err := sess.Exec(updateSQL, encrypted, nowInUTC(), row.Id)
return err
}); err != nil {
logger.Warn("Could not update secret while re-encrypting it", "table", s.tableName, "id", row.Id, "error", err)
return err
}
@ -88,7 +92,7 @@ func (s b64Secret) reencrypt(ctx context.Context, secretsSrv *manager.SecretsSer
continue
}
err := sqlStore.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
err := sqlStore.InTransaction(ctx, func(ctx context.Context) error {
decoded, err := s.encoding.DecodeString(row.Secret)
if err != nil {
logger.Warn("Could not decode base64-encoded secret while re-encrypting it", "table", s.tableName, "id", row.Id, "error", err)
@ -101,22 +105,23 @@ func (s b64Secret) reencrypt(ctx context.Context, secretsSrv *manager.SecretsSer
return err
}
encrypted, err := secretsSrv.EncryptWithDBSession(ctx, decrypted, secrets.WithoutScope(), sess.Session)
encrypted, err := secretsSrv.Encrypt(ctx, decrypted, secrets.WithoutScope())
if err != nil {
logger.Warn("Could not encrypt secret while re-encrypting it", "table", s.tableName, "id", row.Id, "error", err)
return err
}
encoded := s.encoding.EncodeToString(encrypted)
if s.hasUpdatedColumn {
updateSQL := fmt.Sprintf("UPDATE %s SET %s = ?, updated = ? WHERE id = ?", s.tableName, s.columnName)
_, err = sess.Exec(updateSQL, encoded, nowInUTC(), row.Id)
} else {
updateSQL := fmt.Sprintf("UPDATE %s SET %s = ? WHERE id = ?", s.tableName, s.columnName)
_, err = sess.Exec(updateSQL, encoded, row.Id)
}
if err != nil {
if err = sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) (err error) {
encoded := s.encoding.EncodeToString(encrypted)
if s.hasUpdatedColumn {
updateSQL := fmt.Sprintf("UPDATE %s SET %s = ?, updated = ? WHERE id = ?", s.tableName, s.columnName)
_, err = sess.Exec(updateSQL, encoded, nowInUTC(), row.Id)
} else {
updateSQL := fmt.Sprintf("UPDATE %s SET %s = ? WHERE id = ?", s.tableName, s.columnName)
_, err = sess.Exec(updateSQL, encoded, row.Id)
}
return
}); err != nil {
logger.Warn("Could not update secret while re-encrypting it", "table", s.tableName, "id", row.Id, "error", err)
return err
}
@ -158,7 +163,7 @@ func (s jsonSecret) reencrypt(ctx context.Context, secretsSrv *manager.SecretsSe
continue
}
err := sqlStore.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
err := sqlStore.InTransaction(ctx, func(ctx context.Context) error {
decrypted, err := secretsSrv.DecryptJsonData(ctx, row.SecureJsonData)
if err != nil {
logger.Warn("Could not decrypt secrets while re-encrypting them", "table", s.tableName, "id", row.Id, "error", err)
@ -170,13 +175,16 @@ func (s jsonSecret) reencrypt(ctx context.Context, secretsSrv *manager.SecretsSe
Updated string
}{Updated: nowInUTC()}
toUpdate.SecureJsonData, err = secretsSrv.EncryptJsonDataWithDBSession(ctx, decrypted, secrets.WithoutScope(), sess.Session)
toUpdate.SecureJsonData, err = secretsSrv.EncryptJsonData(ctx, decrypted, secrets.WithoutScope())
if err != nil {
logger.Warn("Could not re-encrypt secrets", "table", s.tableName, "id", row.Id, "error", err)
return err
}
if _, err := sess.Table(s.tableName).Where("id = ?", row.Id).Update(toUpdate); err != nil {
if err := sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
_, err := sess.Table(s.tableName).Where("id = ?", row.Id).Update(toUpdate)
return err
}); err != nil {
logger.Warn("Could not update secrets while re-encrypting them", "table", s.tableName, "id", row.Id, "error", err)
return err
}
@ -217,7 +225,7 @@ func (s alertingSecret) reencrypt(ctx context.Context, secretsSrv *manager.Secre
for _, result := range results {
result := result
err := sqlStore.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
err := sqlStore.InTransaction(ctx, func(ctx context.Context) error {
postableUserConfig, err := notifier.Load([]byte(result.AlertmanagerConfiguration))
if err != nil {
logger.Warn("Could not load alert_configuration while re-encrypting it", "id", result.Id, "error", err)
@ -239,7 +247,7 @@ func (s alertingSecret) reencrypt(ctx context.Context, secretsSrv *manager.Secre
return err
}
reencrypted, err := secretsSrv.EncryptWithDBSession(ctx, decrypted, secrets.WithoutScope(), sess.Session)
reencrypted, err := secretsSrv.Encrypt(ctx, decrypted, secrets.WithoutScope())
if err != nil {
logger.Warn("Could not re-encrypt alert_configuration secret", "id", result.Id, "key", k, "error", err)
return err
@ -257,7 +265,10 @@ func (s alertingSecret) reencrypt(ctx context.Context, secretsSrv *manager.Secre
}
result.AlertmanagerConfiguration = string(marshalled)
if _, err := sess.Table("alert_configuration").Where("id = ?", result.Id).Update(&result); err != nil {
if err := sqlStore.WithDbSession(ctx, func(sess *db.Session) error {
_, err := sess.Table("alert_configuration").Where("id = ?", result.Id).Update(&result)
return err
}); err != nil {
logger.Warn("Could not update alert_configuration secret while re-encrypting it", "id", result.Id, "error", err)
return err
}

3
pkg/services/secrets/secrets.go
View File

@ -5,8 +5,6 @@ import (
"fmt"
"strings"
"time"
"xorm.io/xorm"
)
// Service is an envelope encryption service in charge of encrypting/decrypting secrets.
@ -36,7 +34,6 @@ type Store interface {
GetCurrentDataKey(ctx context.Context, label string) (*DataKey, error)
GetAllDataKeys(ctx context.Context) ([]*DataKey, error)
CreateDataKey(ctx context.Context, dataKey *DataKey) error
CreateDataKeyWithDBSession(ctx context.Context, dataKey *DataKey, sess *xorm.Session) error
DisableDataKeys(ctx context.Context) error
DeleteDataKey(ctx context.Context, id string) error
ReEncryptDataKeys(ctx context.Context, providers map[ProviderID]Provider, currProvider ProviderID) error