IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2024-11-28 19:54:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Usage Stats Updates (#39235)

Browse Source 
* add randomly-generated anonymous id to usage reports
* include uptime in stats
* add last_sent tracking, remove metricsLogger
This commit is contained in:
Dan Cech 2021-09-15 11:47:44 -04:00 committed by GitHub
parent 74beb9a64c
commit f8ae71af5b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 113 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
pkg/infra/usagestats/service.go
View File

@ -6,6 +6,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/login/social"
"github.com/grafana/grafana/pkg/plugins"
@ -15,8 +16,6 @@ import (
"github.com/grafana/grafana/pkg/setting"
)
var metricsLogger log.Logger = log.New("metrics")
type UsageStats interface {
GetUsageReport(context.Context) (UsageReport, error)
RegisterMetricsFunc(MetricsFunc)
@ -33,6 +32,7 @@ type UsageStatsService struct {
PluginManager plugins.Manager
SocialService social.Service
grafanaLive *live.GrafanaLive
kvStore *kvstore.NamespacedKVStore
log log.Logger
@ -40,6 +40,7 @@ type UsageStatsService struct {
externalMetrics []MetricsFunc
concurrentUserStatsCache memoConcurrentUserStats
liveStats liveUsageStats
startTime time.Time
}
type liveUsageStats struct {
@ -54,7 +55,8 @@ type liveUsageStats struct {
func ProvideService(cfg *setting.Cfg, bus bus.Bus, sqlStore *sqlstore.SQLStore,
alertingStats alerting.UsageStatsQuerier, pluginManager plugins.Manager,
socialService social.Service, grafanaLive *live.GrafanaLive) *UsageStatsService {
socialService social.Service, grafanaLive *live.GrafanaLive,
kvStore kvstore.KVStore) *UsageStatsService {
s := &UsageStatsService{
Cfg: cfg,
Bus: bus,
@ -63,7 +65,9 @@ func ProvideService(cfg *setting.Cfg, bus bus.Bus, sqlStore *sqlstore.SQLStore,
oauthProviders: socialService.GetOAuthProviders(),
PluginManager: pluginManager,
grafanaLive: grafanaLive,
kvStore: kvstore.WithNamespace(kvStore, 0, "infra.usagestats"),
log: log.New("infra.usagestats"),
startTime: time.Now(),
}
return s
}
@ -71,7 +75,26 @@ func ProvideService(cfg *setting.Cfg, bus bus.Bus, sqlStore *sqlstore.SQLStore,
func (uss *UsageStatsService) Run(ctx context.Context) error {
uss.updateTotalStats()
sendReportTicker := time.NewTicker(time.Hour * 24)
// try to load last sent time from kv store
lastSent := time.Now()
if val, ok, err := uss.kvStore.Get(ctx, "last_sent"); err != nil {
uss.log.Error("Failed to get last sent time", "error", err)
} else if ok {
if parsed, err := time.Parse(time.RFC3339, val); err != nil {
uss.log.Error("Failed to parse last sent time", "error", err)
} else {
lastSent = parsed
}
}
// calculate initial send delay
sendInterval := time.Hour * 24
nextSendInterval := time.Until(lastSent.Add(sendInterval))
if nextSendInterval < time.Minute {
nextSendInterval = time.Minute
}
sendReportTicker := time.NewTicker(nextSendInterval)
updateStatsTicker := time.NewTicker(time.Minute * 30)
defer sendReportTicker.Stop()
@ -81,8 +104,19 @@ func (uss *UsageStatsService) Run(ctx context.Context) error {
select {
case <-sendReportTicker.C:
if err := uss.sendUsageStats(ctx); err != nil {
metricsLogger.Warn("Failed to send usage stats", "err", err)
uss.log.Warn("Failed to send usage stats", "error", err)
}
lastSent = time.Now()
if err := uss.kvStore.Set(ctx, "last_sent", lastSent.Format(time.RFC3339)); err != nil {
uss.log.Warn("Failed to update last sent time", "error", err)
}
if nextSendInterval != sendInterval {
nextSendInterval = sendInterval
sendReportTicker.Reset(nextSendInterval)
}
// always reset live stats every report tick
uss.resetLiveStats()
case <-updateStatsTicker.C:

63
pkg/infra/usagestats/usage_stats.go
View File

@ -10,6 +10,7 @@ import (
"strings"
"time"
"github.com/google/uuid"
"github.com/grafana/grafana/pkg/infra/metrics"
"github.com/grafana/grafana/pkg/models"
)
@ -24,6 +25,7 @@ type UsageReport struct {
Edition string `json:"edition"`
HasValidLicense bool `json:"hasValidLicense"`
Packaging string `json:"packaging"`
UsageStatsId string `json:"usageStatsId"`
}
func (uss *UsageStatsService) GetUsageReport(ctx context.Context) (UsageReport, error) {
@ -42,11 +44,12 @@ func (uss *UsageStatsService) GetUsageReport(ctx context.Context) (UsageReport,
Arch: runtime.GOARCH,
Edition: edition,
Packaging: uss.Cfg.Packaging,
UsageStatsId: uss.GetUsageStatsId(ctx),
}
statsQuery := models.GetSystemStatsQuery{}
if err := uss.Bus.Dispatch(&statsQuery); err != nil {
metricsLogger.Error("Failed to get system stats", "error", err)
uss.log.Error("Failed to get system stats", "error", err)
return report, err
}
@ -132,7 +135,7 @@ func (uss *UsageStatsService) GetUsageReport(ctx context.Context) (UsageReport,
dsStats := models.GetDataSourceStatsQuery{}
if err := uss.Bus.Dispatch(&dsStats); err != nil {
metricsLogger.Error("Failed to get datasource stats", "error", err)
uss.log.Error("Failed to get datasource stats", "error", err)
return report, err
}
@ -151,7 +154,7 @@ func (uss *UsageStatsService) GetUsageReport(ctx context.Context) (UsageReport,
esDataSourcesQuery := models.GetDataSourcesByTypeQuery{Type: models.DS_ES}
if err := uss.Bus.Dispatch(&esDataSourcesQuery); err != nil {
metricsLogger.Error("Failed to get elasticsearch json data", "error", err)
uss.log.Error("Failed to get elasticsearch json data", "error", err)
return report, err
}
@ -196,7 +199,7 @@ func (uss *UsageStatsService) GetUsageReport(ctx context.Context) (UsageReport,
// fetch datasource access stats
dsAccessStats := models.GetDataSourceAccessStatsQuery{}
if err := uss.Bus.Dispatch(&dsAccessStats); err != nil {
metricsLogger.Error("Failed to get datasource access stats", "error", err)
uss.log.Error("Failed to get datasource access stats", "error", err)
return report, err
}
@ -225,8 +228,8 @@ func (uss *UsageStatsService) GetUsageReport(ctx context.Context) (UsageReport,
// get stats about alert notifier usage
anStats := models.GetAlertNotifierUsageStatsQuery{}
if err := uss.Bus.Dispatch(&anStats); err != nil {
metricsLogger.Error("Failed to get alert notification stats", "error", err)
if err := uss.Bus.DispatchCtx(ctx, &anStats); err != nil {
uss.log.Error("Failed to get alert notification stats", "error", err)
return report, err
}
@ -256,7 +259,7 @@ func (uss *UsageStatsService) GetUsageReport(ctx context.Context) (UsageReport,
// Get concurrent users stats as histogram
concurrentUsersStats, err := uss.GetConcurrentUsersStats(ctx)
if err != nil {
metricsLogger.Error("Failed to get concurrent users stats", "error", err)
uss.log.Error("Failed to get concurrent users stats", "error", err)
return report, err
}
@ -268,6 +271,8 @@ func (uss *UsageStatsService) GetUsageReport(ctx context.Context) (UsageReport,
metrics["stats.auth_token_per_user_le_15"] = concurrentUsersStats.BucketLE15
metrics["stats.auth_token_per_user_le_inf"] = concurrentUsersStats.BucketLEInf
metrics["stats.uptime"] = int64(time.Since(uss.startTime).Seconds())
return report, nil
}
@ -275,7 +280,7 @@ func (uss *UsageStatsService) registerExternalMetrics(metrics map[string]interfa
for _, fn := range uss.externalMetrics {
fnMetrics, err := fn()
if err != nil {
metricsLogger.Error("Failed to fetch external metrics", "error", err)
uss.log.Error("Failed to fetch external metrics", "error", err)
continue
}
@ -294,7 +299,7 @@ func (uss *UsageStatsService) sendUsageStats(ctx context.Context) error {
return nil
}
metricsLogger.Debug(fmt.Sprintf("Sending anonymous usage stats to %s", usageStatsURL))
uss.log.Debug(fmt.Sprintf("Sending anonymous usage stats to %s", usageStatsURL))
report, err := uss.GetUsageReport(ctx)
if err != nil {
@ -307,23 +312,23 @@ func (uss *UsageStatsService) sendUsageStats(ctx context.Context) error {
}
data := bytes.NewBuffer(out)
sendUsageStats(data)
sendUsageStats(uss, data)
return nil
}
// sendUsageStats sends usage statistics.
//
// Stubbable by tests.
var sendUsageStats = func(data *bytes.Buffer) {
var sendUsageStats = func(uss *UsageStatsService, data *bytes.Buffer) {
go func() {
client := http.Client{Timeout: 5 * time.Second}
resp, err := client.Post(usageStatsURL, "application/json", data)
if err != nil {
metricsLogger.Error("Failed to send usage stats", "err", err)
uss.log.Error("Failed to send usage stats", "err", err)
return
}
if err := resp.Body.Close(); err != nil {
metricsLogger.Warn("Failed to close response body", "err", err)
uss.log.Warn("Failed to close response body", "err", err)
}
}()
}
@ -363,7 +368,7 @@ func (uss *UsageStatsService) updateTotalStats() {
statsQuery := models.GetSystemStatsQuery{}
if err := uss.Bus.Dispatch(&statsQuery); err != nil {
metricsLogger.Error("Failed to get system stats", "error", err)
uss.log.Error("Failed to get system stats", "error", err)
return
}
@ -387,7 +392,7 @@ func (uss *UsageStatsService) updateTotalStats() {
dsStats := models.GetDataSourceStatsQuery{}
if err := uss.Bus.Dispatch(&dsStats); err != nil {
metricsLogger.Error("Failed to get datasource stats", "error", err)
uss.log.Error("Failed to get datasource stats", "error", err)
return
}
@ -404,3 +409,31 @@ func (uss *UsageStatsService) ShouldBeReported(dsType string) bool {
return ds.Signature.IsValid() || ds.Signature.IsInternal()
}
func (uss *UsageStatsService) GetUsageStatsId(ctx context.Context) string {
anonId, ok, err := uss.kvStore.Get(ctx, "anonymous_id")
if err != nil {
uss.log.Error("Failed to get usage stats id", "error", err)
return ""
}
if ok {
return anonId
}
newId, err := uuid.NewRandom()
if err != nil {
uss.log.Error("Failed to generate usage stats id", "error", err)
return ""
}
anonId = newId.String()
err = uss.kvStore.Set(ctx, "anonymous_id", anonId)
if err != nil {
uss.log.Error("Failed to store usage stats id", "error", err)
return ""
}
return anonId
}

4
pkg/infra/usagestats/usage_stats_service_test.go
View File

@ -9,6 +9,8 @@ import (
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/util"
"github.com/stretchr/testify/assert"
@ -20,6 +22,8 @@ func TestUsageStatsService_GetConcurrentUsersStats(t *testing.T) {
uss := &UsageStatsService{
Bus: bus.New(),
SQLStore: sqlStore,
kvStore: kvstore.WithNamespace(kvstore.ProvideService(sqlStore), 0, "infra.usagestats"),
log: log.New("infra.usagestats"),
}
createConcurrentTokens(t, sqlStore)

18
pkg/infra/usagestats/usage_stats_test.go
View File

@ -14,6 +14,8 @@ import (
"github.com/grafana/grafana/pkg/api/routing"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/manager"
@ -219,7 +221,7 @@ func TestMetrics(t *testing.T) {
sendUsageStats = origSendUsageStats
})
statsSent := false
sendUsageStats = func(*bytes.Buffer) {
sendUsageStats = func(uss *UsageStatsService, b *bytes.Buffer) {
statsSent = true
}
@ -307,6 +309,10 @@ func TestMetrics(t *testing.T) {
assert.Equal(t, runtime.GOOS, j.Get("os").MustString())
assert.Equal(t, runtime.GOARCH, j.Get("arch").MustString())
usageId := uss.GetUsageStatsId(context.Background())
assert.NotEmpty(t, usageId)
assert.Equal(t, usageId, j.Get("usageStatsId").MustString())
metrics := j.Get("metrics")
assert.Equal(t, getSystemStatsQuery.Result.Dashboards, metrics.Get("stats.dashboards.count").MustInt64())
assert.Equal(t, getSystemStatsQuery.Result.Users, metrics.Get("stats.users.count").MustInt64())
@ -393,6 +399,9 @@ func TestMetrics(t *testing.T) {
assert.Equal(t, 4, metrics.Get("stats.auth_token_per_user_le_12").MustInt())
assert.Equal(t, 5, metrics.Get("stats.auth_token_per_user_le_15").MustInt())
assert.Equal(t, 6, metrics.Get("stats.auth_token_per_user_le_inf").MustInt())
assert.LessOrEqual(t, 60, metrics.Get("stats.uptime").MustInt())
assert.Greater(t, 70, metrics.Get("stats.uptime").MustInt())
})
})
@ -629,14 +638,19 @@ type httpResp struct {
func createService(t *testing.T, cfg setting.Cfg) *UsageStatsService {
t.Helper()
sqlStore := sqlstore.InitTestDB(t)
return &UsageStatsService{
Bus: bus.New(),
Cfg: &cfg,
SQLStore: sqlstore.InitTestDB(t),
SQLStore: sqlStore,
AlertingUsageStats: &alertingUsageMock{},
externalMetrics: make([]MetricsFunc, 0),
PluginManager: &fakePluginManager{},
grafanaLive: newTestLive(t),
kvStore: kvstore.WithNamespace(kvstore.ProvideService(sqlStore), 0, "infra.usagestats"),
log: log.New("infra.usagestats"),
startTime: time.Now().Add(-1 * time.Minute),
}
}