Security: Add secrets service (#39418)

* Add secrets service

* Revert accidental changes in util encryption

* Make minor changes

Move functional options to models

Revert renaming types to models

* Add context

* Minor change in GetDataKey

* Use CreateDataKeyWithDBSession in CreateDataKey

* Handle empty DEK name in DeleteDataKey

* Rename defaultProvider

* Remove secrets store service
This commit is contained in:
Tania B 2021-10-01 15:39:57 +03:00 committed by GitHub
parent a6a3ef74be
commit 62689ec804
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 594 additions and 2 deletions

View File

@ -20,6 +20,7 @@ import (
"github.com/grafana/grafana/pkg/services/notifications"
"github.com/grafana/grafana/pkg/services/provisioning"
"github.com/grafana/grafana/pkg/services/rendering"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor"
"github.com/grafana/grafana/pkg/tsdb/cloudwatch"
"github.com/grafana/grafana/pkg/tsdb/elasticsearch"
@ -46,7 +47,7 @@ func ProvideBackgroundServiceRegistry(
// Need to make sure these are initialized, is there a better place to put them?
_ *azuremonitor.Service, _ *cloudwatch.CloudWatchService, _ *elasticsearch.Service, _ *graphite.Service,
_ *influxdb.Service, _ *loki.Service, _ *opentsdb.Service, _ *prometheus.Service, _ *tempo.Service,
_ *testdatasource.TestDataPlugin, _ *plugindashboards.Service, _ *dashboardsnapshots.Service,
_ *testdatasource.TestDataPlugin, _ *plugindashboards.Service, _ *dashboardsnapshots.Service, _ secrets.SecretsService,
_ *postgres.Service, _ *mysql.Service, _ *mssql.Service, _ *grafanads.Service,
) *BackgroundServiceRegistry {

View File

@ -49,6 +49,7 @@ import (
"github.com/grafana/grafana/pkg/services/rendering"
"github.com/grafana/grafana/pkg/services/schemaloader"
"github.com/grafana/grafana/pkg/services/search"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/shorturls"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
@ -142,6 +143,7 @@ var wireBasicSet = wire.NewSet(
graphite.ProvideService,
prometheus.ProvideService,
elasticsearch.ProvideService,
secrets.ProvideSecretsService,
grafanads.ProvideService,
dashboardsnapshots.ProvideService,
)

View File

@ -0,0 +1,78 @@
package secrets
import (
"context"
"fmt"
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/secrets/types"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
const dataKeysTable = "data_keys"
var logger = log.New("secrets-store")
func (s *SecretsService) GetDataKey(ctx context.Context, name string) (*types.DataKey, error) {
dataKey := &types.DataKey{}
var exists bool
err := s.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
var err error
exists, err = sess.Table(dataKeysTable).
Where("name = ? AND active = ?", name, s.sqlStore.Dialect.BooleanStr(true)).
Get(dataKey)
return err
})
if !exists {
return nil, types.ErrDataKeyNotFound
}
if err != nil {
logger.Error("Failed getting data key", "err", err, "name", name)
return nil, fmt.Errorf("failed getting data key: %w", err)
}
return dataKey, nil
}
func (s *SecretsService) GetAllDataKeys(ctx context.Context) ([]*types.DataKey, error) {
result := make([]*types.DataKey, 0)
err := s.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
err := sess.Table(dataKeysTable).Find(&result)
return err
})
return result, err
}
func (s *SecretsService) CreateDataKey(ctx context.Context, dataKey types.DataKey) error {
return s.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
return s.CreateDataKeyWithDBSession(ctx, dataKey, sess)
})
}
func (s *SecretsService) CreateDataKeyWithDBSession(_ context.Context, dataKey types.DataKey, sess *sqlstore.DBSession) error {
if !dataKey.Active {
return fmt.Errorf("cannot insert deactivated data keys")
}
dataKey.Created = time.Now()
dataKey.Updated = dataKey.Created
_, err := sess.Table(dataKeysTable).Insert(&dataKey)
return err
}
func (s *SecretsService) DeleteDataKey(ctx context.Context, name string) error {
if len(name) == 0 {
return fmt.Errorf("data key name is missing")
}
return s.sqlStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
_, err := sess.Table(dataKeysTable).Delete(&types.DataKey{Name: name})
return err
})
}

View File

@ -0,0 +1,28 @@
package secrets
import (
"github.com/grafana/grafana/pkg/services/encryption"
"github.com/grafana/grafana/pkg/setting"
)
type grafanaProvider struct {
settings setting.Provider
encryption encryption.Service
}
func newGrafanaProvider(settings setting.Provider, encryption encryption.Service) grafanaProvider {
return grafanaProvider{
settings: settings,
encryption: encryption,
}
}
func (p grafanaProvider) Encrypt(blob []byte) ([]byte, error) {
key := p.settings.KeyValue("security", "secret_key").Value()
return p.encryption.Encrypt(blob, key)
}
func (p grafanaProvider) Decrypt(blob []byte) ([]byte, error) {
key := p.settings.KeyValue("security", "secret_key").Value()
return p.encryption.Decrypt(blob, key)
}

View File

@ -0,0 +1,269 @@
package secrets
import (
"bytes"
"context"
"crypto/rand"
"encoding/base64"
"errors"
"fmt"
"time"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/services/encryption"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/services/secrets/types"
"github.com/grafana/grafana/pkg/setting"
)
const defaultProvider = "secretKey"
type SecretsService struct {
sqlStore *sqlstore.SQLStore
bus bus.Bus
enc encryption.Service
settings setting.Provider
defaultProvider string
providers map[string]Provider
dataKeyCache map[string]dataKeyCacheItem
}
func ProvideSecretsService(sqlStore *sqlstore.SQLStore, bus bus.Bus, enc encryption.Service, settings setting.Provider) SecretsService {
providers := map[string]Provider{
defaultProvider: newGrafanaProvider(settings, enc),
}
s := SecretsService{
sqlStore: sqlStore,
bus: bus,
enc: enc,
settings: settings,
defaultProvider: defaultProvider,
providers: providers,
dataKeyCache: make(map[string]dataKeyCacheItem),
}
return s
}
type dataKeyCacheItem struct {
expiry time.Time
dataKey []byte
}
type Provider interface {
Encrypt(blob []byte) ([]byte, error)
Decrypt(blob []byte) ([]byte, error)
}
var b64 = base64.RawStdEncoding
type EncryptionOptions func() string
// WithoutScope uses a root level data key for encryption (DEK),
// in other words this DEK is not bound to any specific scope (not attached to any user, org, etc.).
func WithoutScope() EncryptionOptions {
return func() string {
return "root"
}
}
// WithScope uses a data key for encryption bound to some specific scope (i.e., user, org, etc.).
// Scope should look like "user:10", "org:1".
func WithScope(scope string) EncryptionOptions {
return func() string {
return scope
}
}
func (s *SecretsService) Encrypt(ctx context.Context, payload []byte, opt EncryptionOptions) ([]byte, error) {
scope := opt()
keyName := fmt.Sprintf("%s/%s@%s", time.Now().Format("2006-01-02"), scope, s.defaultProvider)
dataKey, err := s.dataKey(ctx, keyName)
if err != nil {
if errors.Is(err, types.ErrDataKeyNotFound) {
dataKey, err = s.newDataKey(ctx, keyName, scope)
if err != nil {
return nil, err
}
} else {
return nil, err
}
}
encrypted, err := s.enc.Encrypt(payload, string(dataKey))
if err != nil {
return nil, err
}
prefix := make([]byte, b64.EncodedLen(len(keyName))+2)
b64.Encode(prefix[1:], []byte(keyName))
prefix[0] = '#'
prefix[len(prefix)-1] = '#'
blob := make([]byte, len(prefix)+len(encrypted))
copy(blob, prefix)
copy(blob[len(prefix):], encrypted)
return blob, nil
}
func (s *SecretsService) Decrypt(ctx context.Context, payload []byte) ([]byte, error) {
if len(payload) == 0 {
return nil, fmt.Errorf("unable to decrypt empty payload")
}
var dataKey []byte
if payload[0] != '#' {
secretKey := s.settings.KeyValue("security", "secret_key").Value()
dataKey = []byte(secretKey)
} else {
payload = payload[1:]
endOfKey := bytes.Index(payload, []byte{'#'})
if endOfKey == -1 {
return nil, fmt.Errorf("could not find valid key in encrypted payload")
}
b64Key := payload[:endOfKey]
payload = payload[endOfKey+1:]
key := make([]byte, b64.DecodedLen(len(b64Key)))
_, err := b64.Decode(key, b64Key)
if err != nil {
return nil, err
}
dataKey, err = s.dataKey(ctx, string(key))
if err != nil {
return nil, err
}
}
return s.enc.Decrypt(payload, string(dataKey))
}
func (s *SecretsService) EncryptJsonData(ctx context.Context, kv map[string]string, opt EncryptionOptions) (map[string][]byte, error) {
encrypted := make(map[string][]byte)
for key, value := range kv {
encryptedData, err := s.Encrypt(ctx, []byte(value), opt)
if err != nil {
return nil, err
}
encrypted[key] = encryptedData
}
return encrypted, nil
}
func (s *SecretsService) DecryptJsonData(ctx context.Context, sjd map[string][]byte) (map[string]string, error) {
decrypted := make(map[string]string)
for key, data := range sjd {
decryptedData, err := s.Decrypt(ctx, data)
if err != nil {
return nil, err
}
decrypted[key] = string(decryptedData)
}
return decrypted, nil
}
func (s *SecretsService) GetDecryptedValue(ctx context.Context, sjd map[string][]byte, key, fallback string) string {
if value, ok := sjd[key]; ok {
decryptedData, err := s.Decrypt(ctx, value)
if err != nil {
return fallback
}
return string(decryptedData)
}
return fallback
}
func newRandomDataKey() ([]byte, error) {
rawDataKey := make([]byte, 16)
_, err := rand.Read(rawDataKey)
if err != nil {
return nil, err
}
return rawDataKey, nil
}
// newDataKey creates a new random DEK, caches it and returns its value
func (s *SecretsService) newDataKey(ctx context.Context, name string, scope string) ([]byte, error) {
// 1. Create new DEK
dataKey, err := newRandomDataKey()
if err != nil {
return nil, err
}
provider, exists := s.providers[s.defaultProvider]
if !exists {
return nil, fmt.Errorf("could not find encryption provider '%s'", s.defaultProvider)
}
// 2. Encrypt it
encrypted, err := provider.Encrypt(dataKey)
if err != nil {
return nil, err
}
// 3. Store its encrypted value in db
err = s.CreateDataKey(ctx, types.DataKey{
Active: true, // TODO: right now we never mark a key as deactivated
Name: name,
Provider: s.defaultProvider,
EncryptedData: encrypted,
Scope: scope,
})
if err != nil {
return nil, err
}
// 4. Cache its unencrypted value and return it
s.dataKeyCache[name] = dataKeyCacheItem{
expiry: time.Now().Add(15 * time.Minute),
dataKey: dataKey,
}
return dataKey, nil
}
// dataKey looks up DEK in cache or database, and decrypts it
func (s *SecretsService) dataKey(ctx context.Context, name string) ([]byte, error) {
if item, exists := s.dataKeyCache[name]; exists {
if item.expiry.Before(time.Now()) && !item.expiry.IsZero() {
delete(s.dataKeyCache, name)
} else {
return item.dataKey, nil
}
}
// 1. get encrypted data key from database
dataKey, err := s.GetDataKey(ctx, name)
if err != nil {
return nil, err
}
// 2. decrypt data key
provider, exists := s.providers[dataKey.Provider]
if !exists {
return nil, fmt.Errorf("could not find encryption provider '%s'", dataKey.Provider)
}
decrypted, err := provider.Decrypt(dataKey.EncryptedData)
if err != nil {
return nil, err
}
// 3. cache data key
s.dataKeyCache[name] = dataKeyCacheItem{
expiry: time.Now().Add(15 * time.Minute),
dataKey: decrypted,
}
return decrypted, nil
}

View File

@ -0,0 +1,138 @@
package secrets
import (
"context"
"testing"
"github.com/grafana/grafana/pkg/services/secrets/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSecrets_EnvelopeEncryption(t *testing.T) {
svc := SetupTestService(t)
ctx := context.Background()
t.Run("encrypting with no entity_id should create DEK", func(t *testing.T) {
plaintext := []byte("very secret string")
encrypted, err := svc.Encrypt(context.Background(), plaintext, WithoutScope())
require.NoError(t, err)
decrypted, err := svc.Decrypt(context.Background(), encrypted)
require.NoError(t, err)
assert.Equal(t, plaintext, decrypted)
keys, err := svc.GetAllDataKeys(ctx)
require.NoError(t, err)
assert.Equal(t, len(keys), 1)
})
t.Run("encrypting another secret with no entity_id should use the same DEK", func(t *testing.T) {
plaintext := []byte("another very secret string")
encrypted, err := svc.Encrypt(context.Background(), plaintext, WithoutScope())
require.NoError(t, err)
decrypted, err := svc.Decrypt(context.Background(), encrypted)
require.NoError(t, err)
assert.Equal(t, plaintext, decrypted)
keys, err := svc.GetAllDataKeys(ctx)
require.NoError(t, err)
assert.Equal(t, len(keys), 1)
})
t.Run("encrypting with entity_id provided should create a new DEK", func(t *testing.T) {
plaintext := []byte("some test data")
encrypted, err := svc.Encrypt(context.Background(), plaintext, WithScope("user:100"))
require.NoError(t, err)
decrypted, err := svc.Decrypt(context.Background(), encrypted)
require.NoError(t, err)
assert.Equal(t, plaintext, decrypted)
keys, err := svc.GetAllDataKeys(ctx)
require.NoError(t, err)
assert.Equal(t, len(keys), 2)
})
t.Run("decrypting empty payload should return error", func(t *testing.T) {
_, err := svc.Decrypt(context.Background(), []byte(""))
require.Error(t, err)
assert.Equal(t, "unable to decrypt empty payload", err.Error())
})
t.Run("decrypting legacy secret encrypted with secret key from settings", func(t *testing.T) {
expected := "grafana"
encrypted := []byte{122, 56, 53, 113, 101, 117, 73, 89, 20, 254, 36, 112, 112, 16, 128, 232, 227, 52, 166, 108, 192, 5, 28, 125, 126, 42, 197, 190, 251, 36, 94}
decrypted, err := svc.Decrypt(context.Background(), encrypted)
require.NoError(t, err)
assert.Equal(t, expected, string(decrypted))
})
}
func TestSecretsService_DataKeys(t *testing.T) {
svc := SetupTestService(t)
ctx := context.Background()
dataKey := types.DataKey{
Active: true,
Name: "test1",
Provider: "test",
EncryptedData: []byte{0x62, 0xAF, 0xA1, 0x1A},
}
t.Run("querying for a DEK that does not exist", func(t *testing.T) {
res, err := svc.GetDataKey(ctx, dataKey.Name)
assert.ErrorIs(t, types.ErrDataKeyNotFound, err)
assert.Nil(t, res)
})
t.Run("creating an active DEK", func(t *testing.T) {
err := svc.CreateDataKey(ctx, dataKey)
require.NoError(t, err)
res, err := svc.GetDataKey(ctx, dataKey.Name)
require.NoError(t, err)
assert.Equal(t, dataKey.EncryptedData, res.EncryptedData)
assert.Equal(t, dataKey.Provider, res.Provider)
assert.Equal(t, dataKey.Name, res.Name)
assert.True(t, dataKey.Active)
})
t.Run("creating an inactive DEK", func(t *testing.T) {
k := types.DataKey{
Active: false,
Name: "test2",
Provider: "test",
EncryptedData: []byte{0x62, 0xAF, 0xA1, 0x1A},
}
err := svc.CreateDataKey(ctx, k)
require.Error(t, err)
res, err := svc.GetDataKey(ctx, k.Name)
assert.Equal(t, types.ErrDataKeyNotFound, err)
assert.Nil(t, res)
})
t.Run("deleting DEK when no name provided must fail", func(t *testing.T) {
beforeDelete, err := svc.GetAllDataKeys(ctx)
require.NoError(t, err)
err = svc.DeleteDataKey(ctx, "")
require.Error(t, err)
afterDelete, err := svc.GetAllDataKeys(ctx)
require.NoError(t, err)
assert.Equal(t, beforeDelete, afterDelete)
})
t.Run("deleting a DEK", func(t *testing.T) {
err := svc.DeleteDataKey(ctx, dataKey.Name)
require.NoError(t, err)
res, err := svc.GetDataKey(ctx, dataKey.Name)
assert.Equal(t, types.ErrDataKeyNotFound, err)
assert.Nil(t, res)
})
}

View File

@ -0,0 +1,32 @@
package secrets
import (
"testing"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/services/encryption/ossencryption"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/require"
"gopkg.in/ini.v1"
)
func SetupTestService(t *testing.T) SecretsService {
t.Helper()
defaultKey := "SdlklWklckeLS"
if len(setting.SecretKey) > 0 {
defaultKey = setting.SecretKey
}
raw, err := ini.Load([]byte(`
[security]
secret_key = ` + defaultKey))
require.NoError(t, err)
settings := &setting.OSSImpl{Cfg: &setting.Cfg{Raw: raw}}
return ProvideSecretsService(
sqlstore.InitTestDB(t),
bus.New(),
ossencryption.ProvideService(),
settings,
)
}

View File

@ -0,0 +1,18 @@
package types
import (
"errors"
"time"
)
var ErrDataKeyNotFound = errors.New("data key not found")
type DataKey struct {
Active bool
Name string
Scope string
Provider string
EncryptedData []byte
Created time.Time
Updated time.Time
}

View File

@ -52,6 +52,7 @@ func (*OSSMigrations) AddMigration(mg *Migrator) {
addLiveChannelMigrations(mg)
}
ualert.RerunDashAlertMigration(mg)
addSecretsMigration(mg)
addKVStoreMigrations(mg)
}

View File

@ -0,0 +1,21 @@
package migrations
import "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
func addSecretsMigration(mg *migrator.Migrator) {
dataKeysV1 := migrator.Table{
Name: "data_keys",
Columns: []*migrator.Column{
{Name: "name", Type: migrator.DB_NVarchar, Length: 100, IsPrimaryKey: true},
{Name: "active", Type: migrator.DB_Bool},
{Name: "scope", Type: migrator.DB_NVarchar, Length: 30, Nullable: false},
{Name: "provider", Type: migrator.DB_NVarchar, Length: 50, Nullable: false},
{Name: "encrypted_data", Type: migrator.DB_Blob, Nullable: false},
{Name: "created", Type: migrator.DB_DateTime, Nullable: false},
{Name: "updated", Type: migrator.DB_DateTime, Nullable: false},
},
Indices: []*migrator.Index{},
}
mg.AddMigration("create data_keys table", migrator.NewAddTableMigration(dataKeysV1))
}

View File

@ -15,6 +15,8 @@ import (
const saltLength = 8
// Decrypt decrypts a payload with a given secret.
// Deprecated. Do not use it.
// Use encryption.Service instead.
var Decrypt = func(payload []byte, secret string) ([]byte, error) {
if len(payload) < saltLength {
return nil, fmt.Errorf("unable to compute salt")
@ -47,6 +49,8 @@ var Decrypt = func(payload []byte, secret string) ([]byte, error) {
}
// Encrypt encrypts a payload with a given secret.
// Deprecated. Do not use it.
// Use encryption.Service instead.
var Encrypt = func(payload []byte, secret string) ([]byte, error) {
salt, err := GetRandomString(saltLength)
if err != nil {

View File

@ -28,7 +28,7 @@ func TestEncryption(t *testing.T) {
assert.Equal(t, []byte("grafana"), decrypted)
})
t.Run("decrypting empty payload should not fail", func(t *testing.T) {
t.Run("decrypting empty payload should fail", func(t *testing.T) {
_, err := Decrypt([]byte(""), "1234")
require.Error(t, err)