Secrets: Make the Migrator extensible (#67307)

* [Chore] Remove setting provider from secret service

Co-authored-by: Tania B <yalyna.ts@gmail.com>
Co-authored-by: Joan López de la Franca Beltran <joanjan14@gmail.com>

* Add a ShouldBeRedacted func

Co-authored-by: Tania B <yalyna.ts@gmail.com>
Co-authored-by: Joan López de la Franca Beltran <joanjan14@gmail.com>

* Secrets: Make Migrator extensible

Co-authored-by: Gabriel MABILLE <gamab@users.noreply.github.com>
Co-authored-by: Tania B <yalyna.ts@gmail.com>

* Alerting: Fix tests after refactor

Co-authored-by: Gabriel MABILLE <gamab@users.noreply.github.com>
Co-authored-by: Tania B <yalyna.ts@gmail.com>

* Remove commented code no longer used

* Fix Wire bindings

Co-authored-by: Tania B <yalyna.ts@gmail.com>

* Add constructors to secrets

* Linting

* Undo undesired change

---------

Co-authored-by: gamab <gabi.mabs@gmail.com>
Co-authored-by: Tania B <yalyna.ts@gmail.com>
Co-authored-by: Gabriel MABILLE <gamab@users.noreply.github.com>
This commit is contained in:
Joan López de la Franca Beltran 2023-06-19 23:44:01 +02:00 committed by GitHub
parent a50afe67d3
commit cc65b4d46a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 97 additions and 117 deletions

View File

@ -111,7 +111,6 @@ import (
secretsStore "github.com/grafana/grafana/pkg/services/secrets/kvstore"
secretsMigrations "github.com/grafana/grafana/pkg/services/secrets/kvstore/migrations"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
secretsMigrator "github.com/grafana/grafana/pkg/services/secrets/migrator"
"github.com/grafana/grafana/pkg/services/serviceaccounts"
serviceaccountsmanager "github.com/grafana/grafana/pkg/services/serviceaccounts/manager"
serviceaccountsretriever "github.com/grafana/grafana/pkg/services/serviceaccounts/retriever"
@ -264,8 +263,6 @@ var wireBasicSet = wire.NewSet(
wire.Bind(new(secrets.Service), new(*secretsManager.SecretsService)),
secretsDatabase.ProvideSecretsStore,
wire.Bind(new(secrets.Store), new(*secretsDatabase.SecretsStoreImpl)),
secretsMigrator.ProvideSecretsMigrator,
wire.Bind(new(secrets.Migrator), new(*secretsMigrator.SecretsMigrator)),
grafanads.ProvideService,
wire.Bind(new(dashboardsnapshots.Store), new(*dashsnapstore.DashboardSnapshotStore)),
dashsnapstore.ProvideStore,

View File

@ -35,6 +35,8 @@ import (
publicdashboardsService "github.com/grafana/grafana/pkg/services/publicdashboards/service"
"github.com/grafana/grafana/pkg/services/searchusers"
"github.com/grafana/grafana/pkg/services/searchusers/filters"
"github.com/grafana/grafana/pkg/services/secrets"
secretsMigrator "github.com/grafana/grafana/pkg/services/secrets/migrator"
"github.com/grafana/grafana/pkg/services/sqlstore/migrations"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/services/validations"
@ -88,6 +90,8 @@ var wireExtsBasicSet = wire.NewSet(
wire.Bind(new(publicdashboards.ServiceWrapper), new(*publicdashboardsService.PublicDashboardServiceWrapperImpl)),
caching.ProvideCachingService,
wire.Bind(new(caching.CachingService), new(*caching.OSSCachingService)),
secretsMigrator.ProvideSecretsMigrator,
wire.Bind(new(secrets.Migrator), new(*secretsMigrator.SecretsMigrator)),
)
var wireExtsSet = wire.NewSet(

View File

@ -31,10 +31,7 @@ func TestIntegrationEngineTimeouts(t *testing.T) {
usValidatorMock := &validator.FakeUsageStatsValidator{}
encProvider := encryptionprovider.ProvideEncryptionProvider()
cfg := setting.NewCfg()
settings := &setting.OSSImpl{Cfg: cfg}
encService, err := encryptionservice.ProvideEncryptionService(encProvider, usMock, settings)
encService, err := encryptionservice.ProvideEncryptionService(encProvider, usMock, setting.NewCfg())
require.NoError(t, err)
tracer := tracing.InitializeTracerForTest()

View File

@ -122,10 +122,7 @@ func TestEngineProcessJob(t *testing.T) {
usValidatorMock := &validator.FakeUsageStatsValidator{}
encProvider := encryptionprovider.ProvideEncryptionProvider()
cfg := setting.NewCfg()
settings := &setting.OSSImpl{Cfg: cfg}
encService, err := encryptionservice.ProvideEncryptionService(encProvider, usMock, settings)
encService, err := encryptionservice.ProvideEncryptionService(encProvider, usMock, setting.NewCfg())
require.NoError(t, err)
tracer := tracing.InitializeTracerForTest()

View File

@ -33,9 +33,7 @@ func TestService(t *testing.T) {
usMock := &usagestats.UsageStatsMock{T: t}
encProvider := encryptionprovider.ProvideEncryptionProvider()
settings := &setting.OSSImpl{Cfg: setting.NewCfg()}
encService, err := encryptionservice.ProvideEncryptionService(encProvider, usMock, settings)
encService, err := encryptionservice.ProvideEncryptionService(encProvider, usMock, setting.NewCfg())
require.NoError(t, err)
s := ProvideService(sqlStore.db, encService, nil)

View File

@ -15,7 +15,7 @@ func SetupTestService(tb testing.TB) *Service {
usMock := &usagestats.UsageStatsMock{T: tb}
provider := encryptionprovider.ProvideEncryptionProvider()
settings := &setting.OSSImpl{Cfg: setting.NewCfg()}
settings := setting.NewCfg()
service, err := ProvideEncryptionService(provider, usMock, settings)
require.NoError(tb, err)

View File

@ -26,8 +26,8 @@ const (
type Service struct {
log log.Logger
settingsProvider setting.Provider
usageMetrics usagestats.Service
cfg *setting.Cfg
usageMetrics usagestats.Service
ciphers map[string]encryption.Cipher
deciphers map[string]encryption.Decipher
@ -36,7 +36,7 @@ type Service struct {
func ProvideEncryptionService(
provider encryption.Provider,
usageMetrics usagestats.Service,
settingsProvider setting.Provider,
cfg *setting.Cfg,
) (*Service, error) {
s := &Service{
log: log.New("encryption"),
@ -44,20 +44,17 @@ func ProvideEncryptionService(
ciphers: provider.ProvideCiphers(),
deciphers: provider.ProvideDeciphers(),
usageMetrics: usageMetrics,
settingsProvider: settingsProvider,
usageMetrics: usageMetrics,
cfg: cfg,
}
algorithm := s.settingsProvider.
KeyValue(securitySection, encryptionAlgorithmKey).
algorithm := s.cfg.SectionWithEnvOverrides(securitySection).Key(encryptionAlgorithmKey).
MustString(defaultEncryptionAlgorithm)
if err := s.checkEncryptionAlgorithm(algorithm); err != nil {
return nil, err
}
settingsProvider.RegisterReloadHandler(securitySection, s)
s.registerUsageMetrics()
return s, nil
@ -86,8 +83,7 @@ func (s *Service) checkEncryptionAlgorithm(algorithm string) error {
func (s *Service) registerUsageMetrics() {
s.usageMetrics.RegisterMetricsFunc(func(context.Context) (map[string]interface{}, error) {
algorithm := s.settingsProvider.
KeyValue(securitySection, encryptionAlgorithmKey).
algorithm := s.cfg.SectionWithEnvOverrides(securitySection).Key(encryptionAlgorithmKey).
MustString(defaultEncryptionAlgorithm)
return map[string]interface{}{
@ -174,8 +170,7 @@ func (s *Service) Encrypt(ctx context.Context, payload []byte, secret string) ([
}
}()
algorithm := s.settingsProvider.
KeyValue(securitySection, encryptionAlgorithmKey).
algorithm := s.cfg.SectionWithEnvOverrides(securitySection).Key(encryptionAlgorithmKey).
MustString(defaultEncryptionAlgorithm)
cipher, ok := s.ciphers[algorithm]
@ -237,20 +232,3 @@ func (s *Service) GetDecryptedValue(ctx context.Context, sjd map[string][]byte,
return fallback
}
func (s *Service) Validate(section setting.Section) error {
s.log.Debug("Validating encryption config")
algorithm := section.KeyValue(encryptionAlgorithmKey).
MustString(defaultEncryptionAlgorithm)
if err := s.checkEncryptionAlgorithm(algorithm); err != nil {
return err
}
return nil
}
func (s *Service) Reload(_ setting.Section) error {
return nil
}

View File

@ -18,7 +18,7 @@ func Test_Service(t *testing.T) {
encProvider := provider.Provider{}
usageStats := &usagestats.UsageStatsMock{}
settings := &setting.OSSImpl{Cfg: setting.NewCfg()}
settings := setting.NewCfg()
svc, err := ProvideEncryptionService(encProvider, usageStats, settings)
require.NoError(t, err)
@ -31,7 +31,7 @@ func Test_Service(t *testing.T) {
})
t.Run("encrypt and decrypt with aes-cfb should work", func(t *testing.T) {
settings.Cfg.Raw.Section(securitySection).Key(encryptionAlgorithmKey).SetValue(encryption.AesCfb)
settings.Raw.Section(securitySection).Key(encryptionAlgorithmKey).SetValue(encryption.AesCfb)
encrypted, err := svc.Encrypt(ctx, []byte("grafana"), "1234")
require.NoError(t, err)
@ -55,7 +55,7 @@ func Test_Service(t *testing.T) {
})
t.Run("encrypt with aes-gcm should fail", func(t *testing.T) {
settings.Cfg.Raw.Section(securitySection).Key(encryptionAlgorithmKey).SetValue(encryption.AesGcm)
settings.Raw.Section(securitySection).Key(encryptionAlgorithmKey).SetValue(encryption.AesGcm)
_, err := svc.Encrypt(ctx, []byte("grafana"), "1234")
require.Error(t, err)
@ -77,7 +77,7 @@ func Test_Service(t *testing.T) {
func Test_Service_MissingProvider(t *testing.T) {
encProvider := fakeProvider{}
usageStats := &usagestats.UsageStatsMock{}
settings := &setting.OSSImpl{Cfg: setting.NewCfg()}
settings := setting.NewCfg()
service, err := ProvideEncryptionService(encProvider, usageStats, settings)
assert.Nil(t, service)

View File

@ -9,23 +9,23 @@ import (
)
type grafanaProvider struct {
settings setting.Provider
cfg *setting.Cfg
encryption encryption.Internal
}
func New(settings setting.Provider, encryption encryption.Internal) secrets.Provider {
func New(cfg *setting.Cfg, encryption encryption.Internal) secrets.Provider {
return grafanaProvider{
settings: settings,
cfg: cfg,
encryption: encryption,
}
}
func (p grafanaProvider) Encrypt(ctx context.Context, blob []byte) ([]byte, error) {
key := p.settings.KeyValue("security", "secret_key").Value()
key := p.cfg.SectionWithEnvOverrides("security").Key("secret_key").Value()
return p.encryption.Encrypt(ctx, blob, key)
}
func (p grafanaProvider) Decrypt(ctx context.Context, blob []byte) ([]byte, error) {
key := p.settings.KeyValue("security", "secret_key").Value()
key := p.cfg.SectionWithEnvOverrides("security").Key("secret_key").Value()
return p.encryption.Decrypt(ctx, blob, key)
}

View File

@ -11,20 +11,20 @@ import (
type Service struct {
enc encryption.Internal
settings setting.Provider
cfg *setting.Cfg
features featuremgmt.FeatureToggles
}
func ProvideService(enc encryption.Internal, settings setting.Provider, features featuremgmt.FeatureToggles) Service {
func ProvideService(enc encryption.Internal, cfg *setting.Cfg, features featuremgmt.FeatureToggles) Service {
return Service{
enc: enc,
settings: settings,
cfg: cfg,
features: features,
}
}
func (s Service) Provide() (map[secrets.ProviderID]secrets.Provider, error) {
return map[secrets.ProviderID]secrets.Provider{
kmsproviders.Default: grafana.New(s.settings, s.enc),
kmsproviders.Default: grafana.New(s.cfg, s.enc),
}, nil
}

View File

@ -39,19 +39,18 @@ func setupTestService(tb testing.TB, store secrets.Store, features *featuremgmt.
require.NoError(tb, err)
cfg := &setting.Cfg{Raw: raw}
settings := &setting.OSSImpl{Cfg: cfg}
encProvider := encryptionprovider.Provider{}
usageStats := &usagestats.UsageStatsMock{}
encryption, err := encryptionservice.ProvideEncryptionService(encProvider, usageStats, settings)
encryption, err := encryptionservice.ProvideEncryptionService(encProvider, usageStats, cfg)
require.NoError(tb, err)
secretsService, err := ProvideSecretsService(
store,
osskmsproviders.ProvideService(encryption, settings, features),
osskmsproviders.ProvideService(encryption, cfg, features),
encryption,
settings,
cfg,
features,
&usagestats.UsageStatsMock{T: tb},
)

View File

@ -37,7 +37,7 @@ var (
type SecretsService struct {
store secrets.Store
enc encryption.Internal
settings setting.Provider
cfg *setting.Cfg
features featuremgmt.FeatureToggles
usageStats usagestats.Service
@ -57,20 +57,20 @@ func ProvideSecretsService(
store secrets.Store,
kmsProvidersService kmsproviders.Service,
enc encryption.Internal,
settings setting.Provider,
cfg *setting.Cfg,
features featuremgmt.FeatureToggles,
usageStats usagestats.Service,
) (*SecretsService, error) {
ttl := settings.KeyValue("security.encryption", "data_keys_cache_ttl").MustDuration(15 * time.Minute)
ttl := cfg.SectionWithEnvOverrides("security.encryption").Key("data_keys_cache_ttl").MustDuration(15 * time.Minute)
currentProviderID := kmsproviders.NormalizeProviderID(secrets.ProviderID(
settings.KeyValue("security", "encryption_provider").MustString(kmsproviders.Default),
cfg.SectionWithEnvOverrides("security").Key("encryption_provider").MustString(kmsproviders.Default),
))
s := &SecretsService{
store: store,
enc: enc,
settings: settings,
cfg: cfg,
usageStats: usageStats,
kmsProvidersService: kmsProvidersService,
dataKeyCache: newDataKeyCache(ttl),
@ -342,7 +342,7 @@ func (s *SecretsService) Decrypt(ctx context.Context, payload []byte) ([]byte, e
var dataKey []byte
if !s.encryptedWithEnvelopeEncryption(payload) {
secretKey := s.settings.KeyValue("security", "secret_key").Value()
secretKey := s.cfg.SectionWithEnvOverrides("security").Key("secret_key").Value()
dataKey = []byte(secretKey)
} else {
payload = payload[1:]
@ -491,7 +491,7 @@ func (s *SecretsService) ReEncryptDataKeys(ctx context.Context) error {
func (s *SecretsService) Run(ctx context.Context) error {
gc := time.NewTicker(
s.settings.KeyValue("security.encryption", "data_keys_cache_cleanup_interval").
s.cfg.SectionWithEnvOverrides("security.encryption").Key("data_keys_cache_cleanup_interval").
MustDuration(time.Minute),
)

View File

@ -182,16 +182,16 @@ func TestSecretsService_UseCurrentProvider(t *testing.T) {
raw, err := ini.Load([]byte(rawCfg))
require.NoError(t, err)
settings := &setting.OSSImpl{Cfg: &setting.Cfg{Raw: raw}}
cfg := &setting.Cfg{Raw: raw}
encProvider := encryptionprovider.Provider{}
usageStats := &usagestats.UsageStatsMock{}
encryptionService, err := encryptionservice.ProvideEncryptionService(encProvider, usageStats, settings)
encryptionService, err := encryptionservice.ProvideEncryptionService(encProvider, usageStats, cfg)
require.NoError(t, err)
features := featuremgmt.WithFeatures()
kms := newFakeKMS(osskmsproviders.ProvideService(encryptionService, settings, features))
kms := newFakeKMS(osskmsproviders.ProvideService(encryptionService, cfg, features))
testDB := db.InitTestDB(t)
secretStore := database.ProvideSecretsStore(testDB)
@ -199,7 +199,7 @@ func TestSecretsService_UseCurrentProvider(t *testing.T) {
secretStore,
&kms,
encryptionService,
settings,
cfg,
features,
&usagestats.UsageStatsMock{T: t},
)
@ -217,7 +217,7 @@ func TestSecretsService_UseCurrentProvider(t *testing.T) {
secretStore,
&kms,
encryptionService,
settings,
cfg,
features,
&usagestats.UsageStatsMock{T: t},
)

View File

@ -13,12 +13,18 @@ import (
"github.com/grafana/grafana/pkg/setting"
)
type SecretsRotator interface {
ReEncrypt(context.Context, *manager.SecretsService, db.DB) bool
Rollback(context.Context, *manager.SecretsService, encryption.Internal, db.DB, string) bool
}
type SecretsMigrator struct {
encryptionSrv encryption.Internal
secretsSrv *manager.SecretsService
sqlStore db.DB
settings setting.Provider
features featuremgmt.FeatureToggles
rotators []SecretsRotator
}
func ProvideSecretsMigrator(
@ -28,24 +34,7 @@ func ProvideSecretsMigrator(
settings setting.Provider,
features featuremgmt.FeatureToggles,
) *SecretsMigrator {
return &SecretsMigrator{
encryptionSrv: encryptionSrv,
secretsSrv: service,
sqlStore: sqlStore,
settings: settings,
features: features,
}
}
func (m *SecretsMigrator) ReEncryptSecrets(ctx context.Context) (bool, error) {
err := m.initProvidersIfNeeded()
if err != nil {
return false, err
}
toReencrypt := []interface {
reencrypt(context.Context, *manager.SecretsService, db.DB) bool
}{
rotators := []SecretsRotator{
simpleSecret{tableName: "dashboard_snapshot", columnName: "dashboard_encrypted"},
b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_access_token"}, encoding: base64.StdEncoding},
b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_refresh_token"}, encoding: base64.StdEncoding},
@ -56,10 +45,30 @@ func (m *SecretsMigrator) ReEncryptSecrets(ctx context.Context) (bool, error) {
alertingSecret{},
}
return &SecretsMigrator{
encryptionSrv: encryptionSrv,
secretsSrv: service,
sqlStore: sqlStore,
settings: settings,
features: features,
rotators: rotators,
}
}
func (m *SecretsMigrator) RegisterRotators(rotators ...SecretsRotator) {
m.rotators = append(m.rotators, rotators...)
}
func (m *SecretsMigrator) ReEncryptSecrets(ctx context.Context) (bool, error) {
err := m.initProvidersIfNeeded()
if err != nil {
return false, err
}
var anyFailure bool
for _, r := range toReencrypt {
if success := r.reencrypt(ctx, m.secretsSrv, m.sqlStore); !success {
for _, r := range m.rotators {
if success := r.ReEncrypt(ctx, m.secretsSrv, m.sqlStore); !success {
anyFailure = true
}
}
@ -73,23 +82,10 @@ func (m *SecretsMigrator) RollBackSecrets(ctx context.Context) (bool, error) {
return false, err
}
toRollback := []interface {
rollback(context.Context, *manager.SecretsService, encryption.Internal, db.DB, string) bool
}{
simpleSecret{tableName: "dashboard_snapshot", columnName: "dashboard_encrypted"},
b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_access_token"}, encoding: base64.StdEncoding},
b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_refresh_token"}, encoding: base64.StdEncoding},
b64Secret{simpleSecret: simpleSecret{tableName: "user_auth", columnName: "o_auth_token_type"}, encoding: base64.StdEncoding},
b64Secret{simpleSecret: simpleSecret{tableName: "secrets", columnName: "value"}, hasUpdatedColumn: true, encoding: base64.RawStdEncoding},
jsonSecret{tableName: "data_source"},
jsonSecret{tableName: "plugin_setting"},
alertingSecret{},
}
var anyFailure bool
for _, r := range toRollback {
if failed := r.rollback(ctx,
for _, r := range m.rotators {
if failed := r.Rollback(ctx,
m.secretsSrv,
m.encryptionSrv,
m.sqlStore,
@ -133,12 +129,26 @@ type simpleSecret struct {
columnName string
}
func NewSimpleSecret(tableName, columnName string) simpleSecret {
return simpleSecret{
tableName: tableName,
columnName: columnName,
}
}
type b64Secret struct {
simpleSecret
hasUpdatedColumn bool
encoding *base64.Encoding
}
func NewBase64Secret(simple simpleSecret, encoding *base64.Encoding) b64Secret {
return b64Secret{
simpleSecret: simple,
encoding: encoding,
}
}
type jsonSecret struct {
tableName string
}

View File

@ -13,7 +13,7 @@ import (
"github.com/grafana/grafana/pkg/services/sqlstore"
)
func (s simpleSecret) reencrypt(ctx context.Context, secretsSrv *manager.SecretsService, sqlStore db.DB) bool {
func (s simpleSecret) ReEncrypt(ctx context.Context, secretsSrv *manager.SecretsService, sqlStore db.DB) bool {
var rows []struct {
Id int
Secret []byte
@ -72,7 +72,7 @@ func (s simpleSecret) reencrypt(ctx context.Context, secretsSrv *manager.Secrets
return !anyFailure
}
func (s b64Secret) reencrypt(ctx context.Context, secretsSrv *manager.SecretsService, sqlStore db.DB) bool {
func (s b64Secret) ReEncrypt(ctx context.Context, secretsSrv *manager.SecretsService, sqlStore db.DB) bool {
var rows []struct {
Id int
Secret string
@ -143,7 +143,7 @@ func (s b64Secret) reencrypt(ctx context.Context, secretsSrv *manager.SecretsSer
return !anyFailure
}
func (s jsonSecret) reencrypt(ctx context.Context, secretsSrv *manager.SecretsService, sqlStore db.DB) bool {
func (s jsonSecret) ReEncrypt(ctx context.Context, secretsSrv *manager.SecretsService, sqlStore db.DB) bool {
var rows []struct {
Id int
SecureJsonData map[string][]byte
@ -206,7 +206,7 @@ func (s jsonSecret) reencrypt(ctx context.Context, secretsSrv *manager.SecretsSe
return !anyFailure
}
func (s alertingSecret) reencrypt(ctx context.Context, secretsSrv *manager.SecretsService, sqlStore db.DB) bool {
func (s alertingSecret) ReEncrypt(ctx context.Context, secretsSrv *manager.SecretsService, sqlStore db.DB) bool {
var results []struct {
Id int
AlertmanagerConfiguration string

View File

@ -12,7 +12,7 @@ import (
"github.com/grafana/grafana/pkg/services/secrets/manager"
)
func (s simpleSecret) rollback(
func (s simpleSecret) Rollback(
ctx context.Context,
secretsSrv *manager.SecretsService,
encryptionSrv encryption.Internal,
@ -72,7 +72,7 @@ func (s simpleSecret) rollback(
return anyFailure
}
func (s b64Secret) rollback(
func (s b64Secret) Rollback(
ctx context.Context,
secretsSrv *manager.SecretsService,
encryptionSrv encryption.Internal,
@ -146,7 +146,7 @@ func (s b64Secret) rollback(
return anyFailure
}
func (s jsonSecret) rollback(
func (s jsonSecret) Rollback(
ctx context.Context,
secretsSrv *manager.SecretsService,
encryptionSrv encryption.Internal,
@ -210,7 +210,7 @@ func (s jsonSecret) rollback(
return anyFailure
}
func (s alertingSecret) rollback(
func (s alertingSecret) Rollback(
ctx context.Context,
secretsSrv *manager.SecretsService,
encryptionSrv encryption.Internal,