From fbc44025dc2e8579a82edcde513280e872c38132 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Fri, 25 May 2018 16:00:15 +0200 Subject: [PATCH] add usage stats for datasource access mode --- pkg/metrics/metrics.go | 29 +++++++++++++++ pkg/metrics/metrics_test.go | 56 +++++++++++++++++++++++++++++ pkg/models/stats.go | 10 ++++++ pkg/services/sqlstore/stats.go | 8 +++++ pkg/services/sqlstore/stats_test.go | 12 +++++++ 5 files changed, 115 insertions(+) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 03836abe2ad..3d3cfc2e1b6 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -394,6 +394,35 @@ func sendUsageStats() { } metrics["stats.ds.other.count"] = dsOtherCount + dsAccessStats := models.GetDataSourceAccessStatsQuery{} + if err := bus.Dispatch(&dsAccessStats); err != nil { + metricsLogger.Error("Failed to get datasource access stats", "error", err) + return + } + + // send access counters for each data source + // but ignore any custom data sources + // as sending that name could be sensitive information + dsAccessOtherCount := make(map[string]int64) + for _, dsAccessStat := range dsAccessStats.Result { + if dsAccessStat.Access == "" { + continue + } + + access := strings.ToLower(dsAccessStat.Access) + + if models.IsKnownDataSourcePlugin(dsAccessStat.Type) { + metrics["stats.ds_access."+dsAccessStat.Type+"."+access+".count"] = dsAccessStat.Count + } else { + old := dsAccessOtherCount[access] + dsAccessOtherCount[access] = old + dsAccessStat.Count + } + } + + for access, count := range dsAccessOtherCount { + metrics["stats.ds_access.other."+access+".count"] = count + } + out, _ := json.MarshalIndent(report, "", " ") data := bytes.NewBuffer(out) diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index 77a0aef3f24..8d88e03d106 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -67,6 +67,54 @@ func TestMetrics(t *testing.T) { return nil }) + var getDataSourceAccessStatsQuery *models.GetDataSourceAccessStatsQuery + bus.AddHandler("test", func(query *models.GetDataSourceAccessStatsQuery) error { + query.Result = []*models.DataSourceAccessStats{ + { + Type: models.DS_ES, + Access: "direct", + Count: 1, + }, + { + Type: models.DS_ES, + Access: "proxy", + Count: 2, + }, + { + Type: models.DS_PROMETHEUS, + Access: "proxy", + Count: 3, + }, + { + Type: "unknown_ds", + Access: "proxy", + Count: 4, + }, + { + Type: "unknown_ds2", + Access: "", + Count: 5, + }, + { + Type: "unknown_ds3", + Access: "direct", + Count: 6, + }, + { + Type: "unknown_ds4", + Access: "direct", + Count: 7, + }, + { + Type: "unknown_ds5", + Access: "proxy", + Count: 8, + }, + } + getDataSourceAccessStatsQuery = query + return nil + }) + var wg sync.WaitGroup var responseBuffer *bytes.Buffer var req *http.Request @@ -90,6 +138,7 @@ func TestMetrics(t *testing.T) { Convey("Should not gather stats or call http endpoint", func() { So(getSystemStatsQuery, ShouldBeNil) So(getDataSourceStatsQuery, ShouldBeNil) + So(getDataSourceAccessStatsQuery, ShouldBeNil) So(req, ShouldBeNil) }) }) @@ -107,6 +156,7 @@ func TestMetrics(t *testing.T) { So(getSystemStatsQuery, ShouldNotBeNil) So(getDataSourceStatsQuery, ShouldNotBeNil) + So(getDataSourceAccessStatsQuery, ShouldNotBeNil) So(req, ShouldNotBeNil) So(req.Method, ShouldEqual, http.MethodPost) So(req.Header.Get("Content-Type"), ShouldEqual, "application/json") @@ -142,6 +192,12 @@ func TestMetrics(t *testing.T) { So(metrics.Get("stats.ds."+models.DS_ES+".count").MustInt(), ShouldEqual, 9) So(metrics.Get("stats.ds."+models.DS_PROMETHEUS+".count").MustInt(), ShouldEqual, 10) So(metrics.Get("stats.ds.other.count").MustInt(), ShouldEqual, 11+12) + + So(metrics.Get("stats.ds_access."+models.DS_ES+".direct.count").MustInt(), ShouldEqual, 1) + So(metrics.Get("stats.ds_access."+models.DS_ES+".proxy.count").MustInt(), ShouldEqual, 2) + So(metrics.Get("stats.ds_access."+models.DS_PROMETHEUS+".proxy.count").MustInt(), ShouldEqual, 3) + So(metrics.Get("stats.ds_access.other.direct.count").MustInt(), ShouldEqual, 6+7) + So(metrics.Get("stats.ds_access.other.proxy.count").MustInt(), ShouldEqual, 4+8) }) }) diff --git a/pkg/models/stats.go b/pkg/models/stats.go index 0e497621e14..4cd50d37463 100644 --- a/pkg/models/stats.go +++ b/pkg/models/stats.go @@ -30,6 +30,16 @@ type GetDataSourceStatsQuery struct { Result []*DataSourceStats } +type DataSourceAccessStats struct { + Type string + Access string + Count int64 +} + +type GetDataSourceAccessStatsQuery struct { + Result []*DataSourceAccessStats +} + type AdminStats struct { Users int `json:"users"` Orgs int `json:"orgs"` diff --git a/pkg/services/sqlstore/stats.go b/pkg/services/sqlstore/stats.go index e0bc0e2091e..5634e8feb52 100644 --- a/pkg/services/sqlstore/stats.go +++ b/pkg/services/sqlstore/stats.go @@ -10,6 +10,7 @@ import ( func init() { bus.AddHandler("sql", GetSystemStats) bus.AddHandler("sql", GetDataSourceStats) + bus.AddHandler("sql", GetDataSourceAccessStats) bus.AddHandler("sql", GetAdminStats) bus.AddHandler("sql", GetSystemUserCountStats) } @@ -23,6 +24,13 @@ func GetDataSourceStats(query *m.GetDataSourceStatsQuery) error { return err } +func GetDataSourceAccessStats(query *m.GetDataSourceAccessStatsQuery) error { + var rawSql = `SELECT COUNT(*) as count, type, access FROM data_source GROUP BY type, access` + query.Result = make([]*m.DataSourceAccessStats, 0) + err := x.SQL(rawSql).Find(&query.Result) + return err +} + func GetSystemStats(query *m.GetSystemStatsQuery) error { var rawSql = `SELECT ( diff --git a/pkg/services/sqlstore/stats_test.go b/pkg/services/sqlstore/stats_test.go index c98556a68d3..97f0ca0c43e 100644 --- a/pkg/services/sqlstore/stats_test.go +++ b/pkg/services/sqlstore/stats_test.go @@ -23,5 +23,17 @@ func TestStatsDataAccess(t *testing.T) { err := GetSystemUserCountStats(&query) So(err, ShouldBeNil) }) + + Convey("Get datasource stats should not results in error", func() { + query := m.GetDataSourceStatsQuery{} + err := GetDataSourceStats(&query) + So(err, ShouldBeNil) + }) + + Convey("Get datasource access stats should not results in error", func() { + query := m.GetDataSourceAccessStatsQuery{} + err := GetDataSourceAccessStats(&query) + So(err, ShouldBeNil) + }) }) }