ServiceAccounts: Add background service to check for token leaks (#53658)

* add secret scan service

* add token batching

* Apply suggestions from code review

Co-authored-by: Victor Cinaglia <victor@grafana.com>

* fix: finish constant renaming

Co-authored-by: Victor Cinaglia <victor@grafana.com>
This commit is contained in:
Jo 2022-11-07 10:46:19 +00:00 committed by GitHub
parent b40f192c7e
commit 2ce5dcfb91
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 675 additions and 7 deletions

View File

@ -11,17 +11,23 @@ import (
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/serviceaccounts"
"github.com/grafana/grafana/pkg/services/serviceaccounts/api"
"github.com/grafana/grafana/pkg/services/serviceaccounts/secretscan"
"github.com/grafana/grafana/pkg/setting"
)
const (
metricsCollectionInterval = time.Minute * 30
defaultSecretScanInterval = time.Minute * 5
)
type ServiceAccountsService struct {
store serviceaccounts.Store
log log.Logger
backgroundLog log.Logger
store serviceaccounts.Store
log log.Logger
backgroundLog log.Logger
secretScanService secretscan.Checker
secretScanEnabled bool
secretScanInterval time.Duration
}
func ProvideServiceAccountsService(
@ -48,6 +54,14 @@ func ProvideServiceAccountsService(
serviceaccountsAPI := api.NewServiceAccountsAPI(cfg, s, ac, routeRegister, s.store, permissionService)
serviceaccountsAPI.RegisterAPIEndpoints()
s.secretScanEnabled = cfg.SectionWithEnvOverrides("secretscan").Key("enabled").MustBool(false)
if s.secretScanEnabled {
s.secretScanInterval = cfg.SectionWithEnvOverrides("secretscan").
Key("interval").MustDuration(defaultSecretScanInterval)
s.secretScanService = secretscan.NewService(s.store, cfg)
}
return s, nil
}
@ -61,6 +75,27 @@ func (sa *ServiceAccountsService) Run(ctx context.Context) error {
updateStatsTicker := time.NewTicker(metricsCollectionInterval)
defer updateStatsTicker.Stop()
// Enforce a minimum interval of 1 minute.
if sa.secretScanInterval < time.Minute {
sa.backgroundLog.Warn("secret scan interval is too low, increasing to " +
defaultSecretScanInterval.String())
sa.secretScanInterval = defaultSecretScanInterval
}
tokenCheckTicker := time.NewTicker(sa.secretScanInterval)
if !sa.secretScanEnabled {
tokenCheckTicker.Stop()
} else {
sa.backgroundLog.Debug("enabled token secret check and executing first check")
if err := sa.secretScanService.CheckTokens(ctx); err != nil {
sa.backgroundLog.Warn("Failed to check for leaked tokens", "error", err.Error())
}
defer tokenCheckTicker.Stop()
}
for {
select {
case <-ctx.Done():
@ -77,6 +112,12 @@ func (sa *ServiceAccountsService) Run(ctx context.Context) error {
if _, err := sa.getUsageMetrics(ctx); err != nil {
sa.backgroundLog.Warn("Failed to get usage metrics", "error", err.Error())
}
case <-tokenCheckTicker.C:
sa.backgroundLog.Debug("checking for leaked tokens")
if err := sa.secretScanService.CheckTokens(ctx); err != nil {
sa.backgroundLog.Warn("Failed to check for leaked tokens", "error", err.Error())
}
}
}
}

View File

@ -50,6 +50,13 @@ func (sa *ServiceAccountsService) getUsageMetrics(ctx context.Context) (map[stri
stats["stats.serviceaccounts.count"] = sqlStats.ServiceAccounts
stats["stats.serviceaccounts.tokens.count"] = sqlStats.Tokens
var secretScanEnabled int64 = 0
if sa.secretScanEnabled {
secretScanEnabled = 1
}
stats["stats.serviceaccounts.secret_scan.enabled.count"] = secretScanEnabled
MStatTotalServiceAccountTokens.Set(float64(sqlStats.Tokens))
MStatTotalServiceAccounts.Set(float64(sqlStats.ServiceAccounts))

View File

@ -15,7 +15,7 @@ func Test_UsageStats(t *testing.T) {
ServiceAccounts: 1,
Tokens: 1,
}}
svc := ServiceAccountsService{store: storeMock}
svc := ServiceAccountsService{store: storeMock, secretScanEnabled: true}
err := svc.DeleteServiceAccount(context.Background(), 1, 1)
require.NoError(t, err)
assert.Len(t, storeMock.Calls.DeleteServiceAccount, 1)
@ -25,4 +25,5 @@ func Test_UsageStats(t *testing.T) {
assert.Equal(t, int64(1), stats["stats.serviceaccounts.count"].(int64))
assert.Equal(t, int64(1), stats["stats.serviceaccounts.tokens.count"].(int64))
assert.Equal(t, int64(1), stats["stats.serviceaccounts.secret_scan.enabled.count"].(int64))
}

View File

@ -0,0 +1,135 @@
package secretscan
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/pkg/errors"
)
const timeout = 4 * time.Second
const maxTokensPerRequest = 100
// SecretScan Client is grafana's client for checking leaked keys.
// Don't use this client directly,
// use the secretscan Service which handles token collection and checking instead.
type client struct {
httpClient *http.Client
version string
baseURL string
}
type secretscanRequest struct {
KeyHashes []string `json:"hashes"`
}
type Token struct {
Type string `json:"type"`
URL string `json:"url"`
Hash string `json:"hash"`
ReportedAt string `json:"reported_at"` //nolint
}
var ErrInvalidStatusCode = errors.New("invalid status code")
func newClient(url, version string) *client {
return &client{
version: version,
baseURL: url,
httpClient: &http.Client{
Timeout: timeout,
Transport: nil,
CheckRedirect: nil,
Jar: nil,
},
}
}
// checkTokens checks if any leaked tokens exist.
// Returns list of leaked tokens.
func (c *client) CheckTokens(ctx context.Context, keyHashes []string) ([]Token, error) {
// decode response body
tokens := make([]Token, 0, len(keyHashes))
// batch requests to secretscan server
err := batch(len(keyHashes), maxTokensPerRequest, func(start, end int) error {
bTokens, err := c.checkTokens(ctx, keyHashes[start:end])
if err != nil {
return err
}
tokens = append(tokens, bTokens...)
return nil
})
if err != nil {
return nil, err
}
return tokens, nil
}
func (c *client) checkTokens(ctx context.Context, keyHashes []string) ([]Token, error) {
// create request body
values := secretscanRequest{KeyHashes: keyHashes}
jsonValue, err := json.Marshal(values)
if err != nil {
return nil, errors.Wrap(err, "failed to make http request")
}
// Build URL
url := fmt.Sprintf("%s/tokens", c.baseURL)
// Create request for secretscan server
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
url, bytes.NewReader(jsonValue))
if err != nil {
return nil, errors.Wrap(err, "failed to make http request")
}
// Set headers
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
req.Header.Set("User-Agent", "grafana-secretscan-client/"+c.version)
// make http POST request to check for leaked tokens.
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, errors.Wrap(err, "failed to do http request")
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%w. status code: %s", ErrInvalidStatusCode, resp.Status)
}
// decode response body
var tokens []Token
if err := json.NewDecoder(resp.Body).Decode(&tokens); err != nil {
return nil, errors.Wrap(err, "failed to decode response body")
}
return tokens, nil
}
func batch(count, size int, eachFn func(start, end int) error) error {
for i := 0; i < count; {
end := i + size
if end > count {
end = count
}
if err := eachFn(i, end); err != nil {
return err
}
i = end
}
return nil
}

View File

@ -0,0 +1,66 @@
package secretscan
import (
"context"
"github.com/grafana/grafana/pkg/services/apikey"
"github.com/grafana/grafana/pkg/services/serviceaccounts"
)
type MockTokenRetriever struct {
keys []apikey.APIKey
errList error
errRevoke error
listCalls []interface{}
revokeCalls [][]interface{}
}
func (m *MockTokenRetriever) ListTokens(
ctx context.Context, query *serviceaccounts.GetSATokensQuery,
) ([]apikey.APIKey, error) {
m.listCalls = append(m.listCalls, query)
return m.keys, m.errList
}
func (m *MockTokenRetriever) RevokeServiceAccountToken(
ctx context.Context, orgID, serviceAccountID, tokenID int64,
) error {
m.revokeCalls = append(m.revokeCalls, []interface{}{orgID, serviceAccountID, tokenID})
return m.errRevoke
}
type MockSecretScaner struct{}
func (m *MockSecretScaner) CheckTokens(ctx context.Context) error {
return nil
}
type MockSecretScanClient struct {
tokens []Token
err error
checkCalls []interface{}
}
func (m *MockSecretScanClient) CheckTokens(ctx context.Context, keyHashes []string) ([]Token, error) {
m.checkCalls = append(m.checkCalls, keyHashes)
return m.tokens, m.err
}
type MockSecretScanNotifier struct {
err error
notifyCalls [][]interface{}
}
func (m *MockSecretScanNotifier) Notify(ctx context.Context,
token *Token, tokenName string, revoked bool,
) error {
m.notifyCalls = append(m.notifyCalls, []interface{}{token, tokenName, revoked})
return m.err
}

View File

@ -0,0 +1,147 @@
package secretscan
import (
"context"
"fmt"
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/apikey"
"github.com/grafana/grafana/pkg/services/serviceaccounts"
"github.com/grafana/grafana/pkg/setting"
)
const defaultURL = "https://secretscan.grafana.com"
type Checker interface {
CheckTokens(ctx context.Context) error
}
type CheckerClient interface {
CheckTokens(ctx context.Context, keyHashes []string) ([]Token, error)
}
type WebHookClient interface {
Notify(ctx context.Context, token *Token, tokenName string, revoked bool) error
}
type SATokenRetriever interface {
ListTokens(ctx context.Context, query *serviceaccounts.GetSATokensQuery) ([]apikey.APIKey, error)
RevokeServiceAccountToken(ctx context.Context, orgID, serviceAccountID, tokenID int64) error
}
// Secret Scan Service is grafana's service for checking leaked keys.
type Service struct {
store SATokenRetriever
client CheckerClient
webHookClient WebHookClient
logger log.Logger
webHookNotify bool
revoke bool // whether to revoke leaked tokens
}
func NewService(store SATokenRetriever, cfg *setting.Cfg) *Service {
secretscanBaseURL := cfg.SectionWithEnvOverrides("secretscan").Key("base_url").MustString(defaultURL)
// URL to send outgoing webhook when a token is leaked.
oncallURL := cfg.SectionWithEnvOverrides("secretscan").Key("oncall_url").MustString("")
revoke := cfg.SectionWithEnvOverrides("secretscan").Key("revoke").MustBool(true)
return &Service{
store: store,
client: newClient(secretscanBaseURL, cfg.BuildVersion),
webHookClient: newWebHookClient(oncallURL, cfg.BuildVersion),
logger: log.New("secretscan"),
webHookNotify: oncallURL != "",
revoke: revoke,
}
}
func (s *Service) RetrieveActiveTokens(ctx context.Context) ([]apikey.APIKey, error) {
saTokens, err := s.store.ListTokens(ctx, &serviceaccounts.GetSATokensQuery{})
if err != nil {
return nil, fmt.Errorf("failed to retrieve service account tokens: %w", err)
}
return saTokens, nil
}
// hasExpired returns true if the token has expired.
// Duplicate to SA API. Remerge.
func hasExpired(expiration *int64) bool {
if expiration == nil {
return false
}
v := time.Unix(*expiration, 0)
return (v).Before(time.Now())
}
// CheckTokens checks for leaked tokens.
func (s *Service) CheckTokens(ctx context.Context) error {
// Retrieve all active tokens from the database.
tokens, err := s.RetrieveActiveTokens(ctx)
if err != nil {
return fmt.Errorf("failed to retrieve tokens for checking: %w", err)
}
hashes, hashMap := s.filterCheckableTokens(tokens)
if len(hashes) == 0 {
s.logger.Debug("no active tokens to check")
return nil
}
// Check if any leaked tokens exist.
secretscanTokens, err := s.client.CheckTokens(ctx, hashes)
if err != nil {
return fmt.Errorf("failed to check tokens: %w", err)
}
// Revoke leaked tokens.
// Could be done in bulk but we don't expect more than 1 or 2 tokens to be leaked per check.
for _, secretscanToken := range secretscanTokens {
secretscanToken := secretscanToken
leakedToken := hashMap[secretscanToken.Hash]
if s.revoke {
if err := s.store.RevokeServiceAccountToken(
ctx, leakedToken.OrgId, *leakedToken.ServiceAccountId, leakedToken.Id); err != nil {
s.logger.Error("failed to delete leaked token. Revoke manually.",
"error", err, "url", secretscanToken.URL, "reported_at", secretscanToken.ReportedAt,
"token_id", leakedToken.Id, "token", leakedToken.Name, "org", leakedToken.OrgId,
"serviceAccount", *leakedToken.ServiceAccountId)
}
}
if s.webHookNotify {
if err := s.webHookClient.Notify(ctx, &secretscanToken, leakedToken.Name, s.revoke); err != nil {
s.logger.Warn("failed to call token leak webhook", "error", err)
}
}
s.logger.Warn("found leaked token",
"url", secretscanToken.URL, "reported_at", secretscanToken.ReportedAt,
"token_id", leakedToken.Id, "token", leakedToken.Name, "org", leakedToken.OrgId,
"serviceAccount", *leakedToken.ServiceAccountId, "revoked", s.revoke)
}
return nil
}
// filterCheckableTokens returns a list of tokens that can be checked and a map of tokens to their hashes.
func (*Service) filterCheckableTokens(tokens []apikey.APIKey) ([]string, map[string]apikey.APIKey) {
hashes := make([]string, 0, len(tokens))
hashMap := make(map[string]apikey.APIKey)
for _, token := range tokens {
if hasExpired(token.Expires) || (token.IsRevoked != nil && *token.IsRevoked) {
continue
}
hashes = append(hashes, token.Key)
hashMap[token.Key] = token
}
return hashes, hashMap
}

View File

@ -0,0 +1,171 @@
package secretscan
import (
"context"
"testing"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/apikey"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestService_CheckTokens(t *testing.T) {
type testCase struct {
desc string
retrievedTokens []apikey.APIKey
wantHashes []string
leakedTokens []Token
notify bool
revoke bool
}
ctx := context.Background()
falseBool := false
trueBool := true
testCases := []testCase{
{
desc: "no tokens",
retrievedTokens: []apikey.APIKey{},
leakedTokens: []Token{},
notify: false,
revoke: false,
},
{
desc: "one token leaked - no revoke, no notify",
retrievedTokens: []apikey.APIKey{{
Id: 1,
OrgId: 2,
Name: "test",
Key: "test-hash-1",
Role: "Viewer",
Expires: nil,
ServiceAccountId: new(int64),
IsRevoked: &falseBool,
}},
wantHashes: []string{"test-hash-1"},
leakedTokens: []Token{{Hash: "test-hash-1"}},
notify: false,
revoke: false,
},
{
desc: "one token leaked - revoke, no notify",
retrievedTokens: []apikey.APIKey{{
Id: 1,
OrgId: 2,
Name: "test",
Key: "test-hash-1",
Role: "Viewer",
Expires: nil,
ServiceAccountId: new(int64),
IsRevoked: &falseBool,
}},
wantHashes: []string{"test-hash-1"},
leakedTokens: []Token{{Hash: "test-hash-1"}},
notify: false,
revoke: true,
},
{
desc: "two tokens - one revoke, notify",
retrievedTokens: []apikey.APIKey{{
Id: 1,
OrgId: 2,
Name: "test",
Key: "test-hash-1",
Role: "Viewer",
Expires: nil,
ServiceAccountId: new(int64),
IsRevoked: &falseBool,
}, {
Id: 2,
OrgId: 4,
Name: "test-2",
Key: "test-hash-2",
Role: "Viewer",
Expires: nil,
ServiceAccountId: new(int64),
IsRevoked: &falseBool,
}},
wantHashes: []string{"test-hash-1", "test-hash-2"},
leakedTokens: []Token{{Hash: "test-hash-2"}},
notify: true,
revoke: true,
},
{
desc: "one token already revoked should not be checked",
retrievedTokens: []apikey.APIKey{{
Id: 1,
OrgId: 2,
Name: "test",
Key: "test-hash-1",
Role: "Viewer",
Expires: nil,
ServiceAccountId: new(int64),
IsRevoked: &trueBool,
}},
wantHashes: []string{},
leakedTokens: []Token{},
notify: false,
revoke: true,
},
{
desc: "one token expired should not be checked",
retrievedTokens: []apikey.APIKey{{
Id: 1,
OrgId: 2,
Name: "test",
Key: "test-hash-1",
Role: "Viewer",
Expires: new(int64),
ServiceAccountId: new(int64),
IsRevoked: &falseBool,
}},
wantHashes: []string{},
leakedTokens: []Token{},
notify: false,
revoke: true,
},
}
for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) {
tokenStore := &MockTokenRetriever{keys: tt.retrievedTokens}
client := &MockSecretScanClient{tokens: tt.leakedTokens}
notifier := &MockSecretScanNotifier{}
service := &Service{
store: tokenStore,
client: client,
webHookClient: notifier,
logger: log.New("secretscan"),
webHookNotify: tt.notify,
revoke: tt.revoke,
}
err := service.CheckTokens(ctx)
require.NoError(t, err)
if len(tt.wantHashes) > 0 {
assert.Equal(t, tt.wantHashes, client.checkCalls[0].([]string))
} else {
assert.Empty(t, client.checkCalls)
}
if len(tt.leakedTokens) > 0 {
if tt.revoke {
assert.Len(t, tokenStore.revokeCalls, len(tt.leakedTokens))
} else {
assert.Empty(t, tokenStore.revokeCalls)
}
}
if tt.notify {
assert.Len(t, notifier.notifyCalls, len(tt.leakedTokens))
} else {
assert.Empty(t, notifier.notifyCalls)
}
})
}
}

View File

@ -0,0 +1,88 @@
package secretscan
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"github.com/google/uuid"
"github.com/pkg/errors"
)
// webHookClient is a client for sending leak notifications.
type webHookClient struct {
httpClient *http.Client
version string
url string
}
var ErrInvalidWebHookStatusCode = errors.New("invalid webhook status code")
func newWebHookClient(url, version string) *webHookClient {
return &webHookClient{
version: version,
url: url,
httpClient: &http.Client{
Transport: nil,
CheckRedirect: nil,
Jar: nil,
Timeout: timeout,
},
}
}
func (wClient *webHookClient) Notify(ctx context.Context,
token *Token, tokenName string, revoked bool,
) error {
revokedMsg := ""
if revoked {
revokedMsg = " Grafana has revoked this token"
}
// create request body
values := map[string]interface{}{
"alert_uid": uuid.NewString(),
"title": "SecretScan Alert: Grafana Token leaked",
"image_url": "https://images.pexels.com/photos/5119737/pexels-photo-5119737.jpeg?auto=compress&cs=tinysrgb&w=300", //nolint
"state": "alerting",
"link_to_upstream_details": token.URL,
"message": "Token of type " +
token.Type + " with name " +
tokenName + " has been publicly exposed in " +
token.URL + "." + revokedMsg,
}
jsonValue, err := json.Marshal(values)
if err != nil {
return errors.Wrap(err, "failed to marshal webhook request")
}
// Build URL
// Create request for secretscan server
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
wClient.url, bytes.NewReader(jsonValue))
if err != nil {
return errors.Wrap(err, "failed to make http request")
}
// Set headers
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
req.Header.Set("User-Agent", "grafana-secretscan-webhook-client/"+wClient.version)
// make http POST request to check for leaked tokens.
resp, err := wClient.httpClient.Do(req)
if err != nil {
return errors.Wrap(err, "failed to webhook request")
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("%w. status code %s", ErrInvalidWebHookStatusCode, resp.Status)
}
return nil
}

View File

@ -38,9 +38,7 @@ export const ServiceAccountTokensTable = ({ tokens, timeZone, tokenActionsDisabl
</td>
<td>{formatDate(timeZone, key.created)}</td>
<td>{formatLastUsedAtDate(timeZone, key.lastUsedAt)}</td>
<td className="width-1 text-center">
{key.isRevoked && <span className="label label-tag label-tag--gray">Revoked</span>}
</td>
<td className="width-1 text-center">{key.isRevoked && <TokenRevoked />}</td>
<td>
<DeleteButton
aria-label={`Delete service account token ${key.name}`}
@ -77,6 +75,20 @@ function formatSecondsLeftUntilExpiration(secondsUntilExpiration: number): strin
return `Expires in ${daysFormat}`;
}
const TokenRevoked = () => {
const styles = useStyles2(getStyles);
return (
<span className={styles.hasExpired}>
Revoked
<span className={styles.tooltipContainer}>
<Tooltip content="This token has been publicly exposed. Please rotate this token">
<Icon name="exclamation-triangle" className={styles.toolTipIcon} />
</Tooltip>
</span>
</span>
);
};
interface TokenExpirationProps {
timeZone: TimeZone;
token: ApiKey;