diff --git a/pkg/infra/usagestats/service.go b/pkg/infra/usagestats/service.go new file mode 100644 index 00000000000..c2bf0d06349 --- /dev/null +++ b/pkg/infra/usagestats/service.go @@ -0,0 +1,54 @@ +package usagestats + +import ( + "context" + "time" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/social" + + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/registry" + "github.com/grafana/grafana/pkg/setting" +) + +var metricsLogger log.Logger = log.New("metrics") + +func init() { + registry.RegisterService(&UsageStatsService{}) +} + +type UsageStatsService struct { + Cfg *setting.Cfg `inject:""` + Bus bus.Bus `inject:""` + SQLStore *sqlstore.SqlStore `inject:""` + + oauthProviders map[string]bool +} + +func (uss *UsageStatsService) Init() error { + + uss.oauthProviders = social.GetOAuthProviders(uss.Cfg) + return nil +} + +func (uss *UsageStatsService) Run(ctx context.Context) error { + uss.updateTotalStats() + + onceEveryDayTick := time.NewTicker(time.Hour * 24) + everyMinuteTicker := time.NewTicker(time.Minute) + defer onceEveryDayTick.Stop() + defer everyMinuteTicker.Stop() + + for { + select { + case <-onceEveryDayTick.C: + uss.sendUsageStats(uss.oauthProviders) + case <-everyMinuteTicker.C: + uss.updateTotalStats() + case <-ctx.Done(): + return ctx.Err() + } + } +} diff --git a/pkg/infra/usagestats/usage_stats.go b/pkg/infra/usagestats/usage_stats.go new file mode 100644 index 00000000000..b54de124335 --- /dev/null +++ b/pkg/infra/usagestats/usage_stats.go @@ -0,0 +1,177 @@ +package usagestats + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "runtime" + "strings" + "time" + + "github.com/grafana/grafana/pkg/metrics" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/setting" +) + +var usageStatsURL = "https://stats.grafana.org/grafana-usage-report" + +func (uss *UsageStatsService) sendUsageStats(oauthProviders map[string]bool) { + if !setting.ReportingEnabled { + return + } + + metricsLogger.Debug(fmt.Sprintf("Sending anonymous usage stats to %s", usageStatsURL)) + + version := strings.Replace(setting.BuildVersion, ".", "_", -1) + + metrics := map[string]interface{}{} + report := map[string]interface{}{ + "version": version, + "metrics": metrics, + "os": runtime.GOOS, + "arch": runtime.GOARCH, + "edition": getEdition(), + "packaging": setting.Packaging, + } + + statsQuery := models.GetSystemStatsQuery{} + if err := uss.Bus.Dispatch(&statsQuery); err != nil { + metricsLogger.Error("Failed to get system stats", "error", err) + return + } + + metrics["stats.dashboards.count"] = statsQuery.Result.Dashboards + metrics["stats.users.count"] = statsQuery.Result.Users + metrics["stats.orgs.count"] = statsQuery.Result.Orgs + metrics["stats.playlist.count"] = statsQuery.Result.Playlists + metrics["stats.plugins.apps.count"] = len(plugins.Apps) + metrics["stats.plugins.panels.count"] = len(plugins.Panels) + metrics["stats.plugins.datasources.count"] = len(plugins.DataSources) + metrics["stats.alerts.count"] = statsQuery.Result.Alerts + metrics["stats.active_users.count"] = statsQuery.Result.ActiveUsers + metrics["stats.datasources.count"] = statsQuery.Result.Datasources + metrics["stats.stars.count"] = statsQuery.Result.Stars + metrics["stats.folders.count"] = statsQuery.Result.Folders + metrics["stats.dashboard_permissions.count"] = statsQuery.Result.DashboardPermissions + metrics["stats.folder_permissions.count"] = statsQuery.Result.FolderPermissions + metrics["stats.provisioned_dashboards.count"] = statsQuery.Result.ProvisionedDashboards + metrics["stats.snapshots.count"] = statsQuery.Result.Snapshots + metrics["stats.teams.count"] = statsQuery.Result.Teams + metrics["stats.total_sessions.count"] = statsQuery.Result.Sessions + + userCount := statsQuery.Result.Users + avgSessionsPerUser := statsQuery.Result.Sessions + if userCount != 0 { + avgSessionsPerUser = avgSessionsPerUser / userCount + } + + metrics["stats.avg_sessions_per_user.count"] = avgSessionsPerUser + + dsStats := models.GetDataSourceStatsQuery{} + if err := uss.Bus.Dispatch(&dsStats); err != nil { + metricsLogger.Error("Failed to get datasource stats", "error", err) + return + } + + // send counters for each data source + // but ignore any custom data sources + // as sending that name could be sensitive information + dsOtherCount := 0 + for _, dsStat := range dsStats.Result { + if models.IsKnownDataSourcePlugin(dsStat.Type) { + metrics["stats.ds."+dsStat.Type+".count"] = dsStat.Count + } else { + dsOtherCount += dsStat.Count + } + } + metrics["stats.ds.other.count"] = dsOtherCount + + metrics["stats.packaging."+setting.Packaging+".count"] = 1 + + dsAccessStats := models.GetDataSourceAccessStatsQuery{} + if err := uss.Bus.Dispatch(&dsAccessStats); err != nil { + metricsLogger.Error("Failed to get datasource access stats", "error", err) + return + } + + // send access counters for each data source + // but ignore any custom data sources + // as sending that name could be sensitive information + dsAccessOtherCount := make(map[string]int64) + for _, dsAccessStat := range dsAccessStats.Result { + if dsAccessStat.Access == "" { + continue + } + + access := strings.ToLower(dsAccessStat.Access) + + if models.IsKnownDataSourcePlugin(dsAccessStat.Type) { + metrics["stats.ds_access."+dsAccessStat.Type+"."+access+".count"] = dsAccessStat.Count + } else { + old := dsAccessOtherCount[access] + dsAccessOtherCount[access] = old + dsAccessStat.Count + } + } + + for access, count := range dsAccessOtherCount { + metrics["stats.ds_access.other."+access+".count"] = count + } + + anStats := models.GetAlertNotifierUsageStatsQuery{} + if err := uss.Bus.Dispatch(&anStats); err != nil { + metricsLogger.Error("Failed to get alert notification stats", "error", err) + return + } + + for _, stats := range anStats.Result { + metrics["stats.alert_notifiers."+stats.Type+".count"] = stats.Count + } + + authTypes := map[string]bool{} + authTypes["anonymous"] = setting.AnonymousEnabled + authTypes["basic_auth"] = setting.BasicAuthEnabled + authTypes["ldap"] = setting.LdapEnabled + authTypes["auth_proxy"] = setting.AuthProxyEnabled + + for provider, enabled := range oauthProviders { + authTypes["oauth_"+provider] = enabled + } + + for authType, enabled := range authTypes { + enabledValue := 0 + if enabled { + enabledValue = 1 + } + metrics["stats.auth_enabled."+authType+".count"] = enabledValue + } + + out, _ := json.MarshalIndent(report, "", " ") + data := bytes.NewBuffer(out) + + client := http.Client{Timeout: 5 * time.Second} + go client.Post(usageStatsURL, "application/json", data) +} + +func (uss *UsageStatsService) updateTotalStats() { + statsQuery := models.GetSystemStatsQuery{} + if err := uss.Bus.Dispatch(&statsQuery); err != nil { + metricsLogger.Error("Failed to get system stats", "error", err) + return + } + + metrics.M_StatTotal_Dashboards.Set(float64(statsQuery.Result.Dashboards)) + metrics.M_StatTotal_Users.Set(float64(statsQuery.Result.Users)) + metrics.M_StatActive_Users.Set(float64(statsQuery.Result.ActiveUsers)) + metrics.M_StatTotal_Playlists.Set(float64(statsQuery.Result.Playlists)) + metrics.M_StatTotal_Orgs.Set(float64(statsQuery.Result.Orgs)) +} + +func getEdition() string { + if setting.IsEnterprise { + return "enterprise" + } else { + return "oss" + } +} diff --git a/pkg/metrics/metrics_test.go b/pkg/infra/usagestats/usage_stats_test.go similarity index 91% rename from pkg/metrics/metrics_test.go rename to pkg/infra/usagestats/usage_stats_test.go index c27d6f64b8c..d343ed52b93 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/infra/usagestats/usage_stats_test.go @@ -1,4 +1,4 @@ -package metrics +package usagestats import ( "bytes" @@ -15,14 +15,21 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" . "github.com/smartystreets/goconvey/convey" ) func TestMetrics(t *testing.T) { Convey("Test send usage stats", t, func() { + uss := &UsageStatsService{ + Bus: bus.New(), + SQLStore: sqlstore.InitTestDB(t), + } + var getSystemStatsQuery *models.GetSystemStatsQuery - bus.AddHandler("test", func(query *models.GetSystemStatsQuery) error { + uss.Bus.AddHandler(func(query *models.GetSystemStatsQuery) error { + query.Result = &models.SystemStats{ Dashboards: 1, Datasources: 2, @@ -38,13 +45,14 @@ func TestMetrics(t *testing.T) { ProvisionedDashboards: 12, Snapshots: 13, Teams: 14, + Sessions: 15, } getSystemStatsQuery = query return nil }) var getDataSourceStatsQuery *models.GetDataSourceStatsQuery - bus.AddHandler("test", func(query *models.GetDataSourceStatsQuery) error { + uss.Bus.AddHandler(func(query *models.GetDataSourceStatsQuery) error { query.Result = []*models.DataSourceStats{ { Type: models.DS_ES, @@ -68,7 +76,7 @@ func TestMetrics(t *testing.T) { }) var getDataSourceAccessStatsQuery *models.GetDataSourceAccessStatsQuery - bus.AddHandler("test", func(query *models.GetDataSourceAccessStatsQuery) error { + uss.Bus.AddHandler(func(query *models.GetDataSourceAccessStatsQuery) error { query.Result = []*models.DataSourceAccessStats{ { Type: models.DS_ES, @@ -116,7 +124,7 @@ func TestMetrics(t *testing.T) { }) var getAlertNotifierUsageStatsQuery *models.GetAlertNotifierUsageStatsQuery - bus.AddHandler("test", func(query *models.GetAlertNotifierUsageStatsQuery) error { + uss.Bus.AddHandler(func(query *models.GetAlertNotifierUsageStatsQuery) error { query.Result = []*models.NotifierUsageStats{ { Type: "slack", @@ -155,11 +163,11 @@ func TestMetrics(t *testing.T) { "grafana_com": true, } - sendUsageStats(oauthProviders) + uss.sendUsageStats(oauthProviders) Convey("Given reporting not enabled and sending usage stats", func() { setting.ReportingEnabled = false - sendUsageStats(oauthProviders) + uss.sendUsageStats(oauthProviders) Convey("Should not gather stats or call http endpoint", func() { So(getSystemStatsQuery, ShouldBeNil) @@ -179,7 +187,7 @@ func TestMetrics(t *testing.T) { setting.Packaging = "deb" wg.Add(1) - sendUsageStats(oauthProviders) + uss.sendUsageStats(oauthProviders) Convey("Should gather stats and call http endpoint", func() { if waitTimeout(&wg, 2*time.Second) { @@ -221,6 +229,8 @@ func TestMetrics(t *testing.T) { So(metrics.Get("stats.provisioned_dashboards.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ProvisionedDashboards) So(metrics.Get("stats.snapshots.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Snapshots) So(metrics.Get("stats.teams.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Teams) + So(metrics.Get("stats.total_sessions.count").MustInt64(), ShouldEqual, 15) + So(metrics.Get("stats.avg_sessions_per_user.count").MustInt64(), ShouldEqual, 5) So(metrics.Get("stats.ds."+models.DS_ES+".count").MustInt(), ShouldEqual, 9) So(metrics.Get("stats.ds."+models.DS_PROMETHEUS+".count").MustInt(), ShouldEqual, 10) @@ -246,6 +256,7 @@ func TestMetrics(t *testing.T) { So(metrics.Get("stats.auth_enabled.oauth_grafana_com.count").MustInt(), ShouldEqual, 1) So(metrics.Get("stats.packaging.deb.count").MustInt(), ShouldEqual, 1) + }) }) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 326514a9687..718a63ee768 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -1,17 +1,8 @@ package metrics import ( - "bytes" - "encoding/json" - "net/http" "runtime" - "strings" - "time" - "github.com/grafana/grafana/pkg/bus" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/setting" "github.com/prometheus/client_golang/prometheus" ) @@ -68,23 +59,6 @@ var ( grafanaBuildVersion *prometheus.GaugeVec ) -func newCounterVecStartingAtZero(opts prometheus.CounterOpts, labels []string, labelValues ...string) *prometheus.CounterVec { - counter := prometheus.NewCounterVec(opts, labels) - - for _, label := range labelValues { - counter.WithLabelValues(label).Add(0) - } - - return counter -} - -func newCounterStartingAtZero(opts prometheus.CounterOpts, labelValues ...string) prometheus.Counter { - counter := prometheus.NewCounter(opts) - counter.Add(0) - - return counter -} - func init() { M_Instance_Start = prometheus.NewCounter(prometheus.CounterOpts{ Name: "instance_start_total", @@ -362,154 +336,19 @@ func initMetricVars() { } -func updateTotalStats() { - statsQuery := models.GetSystemStatsQuery{} - if err := bus.Dispatch(&statsQuery); err != nil { - metricsLogger.Error("Failed to get system stats", "error", err) - return +func newCounterVecStartingAtZero(opts prometheus.CounterOpts, labels []string, labelValues ...string) *prometheus.CounterVec { + counter := prometheus.NewCounterVec(opts, labels) + + for _, label := range labelValues { + counter.WithLabelValues(label).Add(0) } - M_StatTotal_Dashboards.Set(float64(statsQuery.Result.Dashboards)) - M_StatTotal_Users.Set(float64(statsQuery.Result.Users)) - M_StatActive_Users.Set(float64(statsQuery.Result.ActiveUsers)) - M_StatTotal_Playlists.Set(float64(statsQuery.Result.Playlists)) - M_StatTotal_Orgs.Set(float64(statsQuery.Result.Orgs)) + return counter } -var usageStatsURL = "https://stats.grafana.org/grafana-usage-report" +func newCounterStartingAtZero(opts prometheus.CounterOpts, labelValues ...string) prometheus.Counter { + counter := prometheus.NewCounter(opts) + counter.Add(0) -func getEdition() string { - if setting.IsEnterprise { - return "enterprise" - } else { - return "oss" - } -} - -func sendUsageStats(oauthProviders map[string]bool) { - if !setting.ReportingEnabled { - return - } - - metricsLogger.Debug("Sending anonymous usage stats to stats.grafana.org") - - version := strings.Replace(setting.BuildVersion, ".", "_", -1) - - metrics := map[string]interface{}{} - report := map[string]interface{}{ - "version": version, - "metrics": metrics, - "os": runtime.GOOS, - "arch": runtime.GOARCH, - "edition": getEdition(), - "packaging": setting.Packaging, - } - - statsQuery := models.GetSystemStatsQuery{} - if err := bus.Dispatch(&statsQuery); err != nil { - metricsLogger.Error("Failed to get system stats", "error", err) - return - } - - metrics["stats.dashboards.count"] = statsQuery.Result.Dashboards - metrics["stats.users.count"] = statsQuery.Result.Users - metrics["stats.orgs.count"] = statsQuery.Result.Orgs - metrics["stats.playlist.count"] = statsQuery.Result.Playlists - metrics["stats.plugins.apps.count"] = len(plugins.Apps) - metrics["stats.plugins.panels.count"] = len(plugins.Panels) - metrics["stats.plugins.datasources.count"] = len(plugins.DataSources) - metrics["stats.alerts.count"] = statsQuery.Result.Alerts - metrics["stats.active_users.count"] = statsQuery.Result.ActiveUsers - metrics["stats.datasources.count"] = statsQuery.Result.Datasources - metrics["stats.stars.count"] = statsQuery.Result.Stars - metrics["stats.folders.count"] = statsQuery.Result.Folders - metrics["stats.dashboard_permissions.count"] = statsQuery.Result.DashboardPermissions - metrics["stats.folder_permissions.count"] = statsQuery.Result.FolderPermissions - metrics["stats.provisioned_dashboards.count"] = statsQuery.Result.ProvisionedDashboards - metrics["stats.snapshots.count"] = statsQuery.Result.Snapshots - metrics["stats.teams.count"] = statsQuery.Result.Teams - - dsStats := models.GetDataSourceStatsQuery{} - if err := bus.Dispatch(&dsStats); err != nil { - metricsLogger.Error("Failed to get datasource stats", "error", err) - return - } - - // send counters for each data source - // but ignore any custom data sources - // as sending that name could be sensitive information - dsOtherCount := 0 - for _, dsStat := range dsStats.Result { - if models.IsKnownDataSourcePlugin(dsStat.Type) { - metrics["stats.ds."+dsStat.Type+".count"] = dsStat.Count - } else { - dsOtherCount += dsStat.Count - } - } - metrics["stats.ds.other.count"] = dsOtherCount - - metrics["stats.packaging."+setting.Packaging+".count"] = 1 - - dsAccessStats := models.GetDataSourceAccessStatsQuery{} - if err := bus.Dispatch(&dsAccessStats); err != nil { - metricsLogger.Error("Failed to get datasource access stats", "error", err) - return - } - - // send access counters for each data source - // but ignore any custom data sources - // as sending that name could be sensitive information - dsAccessOtherCount := make(map[string]int64) - for _, dsAccessStat := range dsAccessStats.Result { - if dsAccessStat.Access == "" { - continue - } - - access := strings.ToLower(dsAccessStat.Access) - - if models.IsKnownDataSourcePlugin(dsAccessStat.Type) { - metrics["stats.ds_access."+dsAccessStat.Type+"."+access+".count"] = dsAccessStat.Count - } else { - old := dsAccessOtherCount[access] - dsAccessOtherCount[access] = old + dsAccessStat.Count - } - } - - for access, count := range dsAccessOtherCount { - metrics["stats.ds_access.other."+access+".count"] = count - } - - anStats := models.GetAlertNotifierUsageStatsQuery{} - if err := bus.Dispatch(&anStats); err != nil { - metricsLogger.Error("Failed to get alert notification stats", "error", err) - return - } - - for _, stats := range anStats.Result { - metrics["stats.alert_notifiers."+stats.Type+".count"] = stats.Count - } - - authTypes := map[string]bool{} - authTypes["anonymous"] = setting.AnonymousEnabled - authTypes["basic_auth"] = setting.BasicAuthEnabled - authTypes["ldap"] = setting.LdapEnabled - authTypes["auth_proxy"] = setting.AuthProxyEnabled - - for provider, enabled := range oauthProviders { - authTypes["oauth_"+provider] = enabled - } - - for authType, enabled := range authTypes { - enabledValue := 0 - if enabled { - enabledValue = 1 - } - metrics["stats.auth_enabled."+authType+".count"] = enabledValue - } - - out, _ := json.MarshalIndent(report, "", " ") - data := bytes.NewBuffer(out) - - client := http.Client{Timeout: 5 * time.Second} - go client.Post(usageStatsURL, "application/json", data) + return counter } diff --git a/pkg/metrics/service.go b/pkg/metrics/service.go index d2c0c815da9..44b83187cac 100644 --- a/pkg/metrics/service.go +++ b/pkg/metrics/service.go @@ -2,7 +2,6 @@ package metrics import ( "context" - "time" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/metrics/graphitebridge" @@ -30,7 +29,6 @@ type InternalMetricsService struct { intervalSeconds int64 graphiteCfg *graphitebridge.Config - oauthProviders map[string]bool } func (im *InternalMetricsService) Init() error { @@ -50,22 +48,6 @@ func (im *InternalMetricsService) Run(ctx context.Context) error { M_Instance_Start.Inc() - // set the total stats gauges before we publishing metrics - updateTotalStats() - - onceEveryDayTick := time.NewTicker(time.Hour * 24) - everyMinuteTicker := time.NewTicker(time.Minute) - defer onceEveryDayTick.Stop() - defer everyMinuteTicker.Stop() - - for { - select { - case <-onceEveryDayTick.C: - sendUsageStats(im.oauthProviders) - case <-everyMinuteTicker.C: - updateTotalStats() - case <-ctx.Done(): - return ctx.Err() - } - } + <-ctx.Done() + return ctx.Err() } diff --git a/pkg/metrics/settings.go b/pkg/metrics/settings.go index 18b9e78d6ff..048e4134690 100644 --- a/pkg/metrics/settings.go +++ b/pkg/metrics/settings.go @@ -5,8 +5,6 @@ import ( "strings" "time" - "github.com/grafana/grafana/pkg/social" - "github.com/grafana/grafana/pkg/metrics/graphitebridge" "github.com/grafana/grafana/pkg/setting" "github.com/prometheus/client_golang/prometheus" @@ -24,8 +22,6 @@ func (im *InternalMetricsService) readSettings() error { return fmt.Errorf("Unable to parse metrics graphite section, %v", err) } - im.oauthProviders = social.GetOAuthProviders(im.Cfg) - return nil } diff --git a/pkg/models/stats.go b/pkg/models/stats.go index d3e145dedf4..00f881f3c59 100644 --- a/pkg/models/stats.go +++ b/pkg/models/stats.go @@ -15,6 +15,7 @@ type SystemStats struct { FolderPermissions int64 Folders int64 ProvisionedDashboards int64 + Sessions int64 } type DataSourceStats struct { diff --git a/pkg/services/sqlstore/stats.go b/pkg/services/sqlstore/stats.go index 2cec86e7239..4c6d6c21221 100644 --- a/pkg/services/sqlstore/stats.go +++ b/pkg/services/sqlstore/stats.go @@ -74,7 +74,8 @@ func GetSystemStats(query *m.GetSystemStatsQuery) error { sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_provisioning") + `) AS provisioned_dashboards,`) sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_snapshot") + `) AS snapshots,`) - sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("team") + `) AS teams`) + sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("team") + `) AS teams,`) + sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("user_auth_token") + `) AS sessions`) var stats m.SystemStats _, err := x.SQL(sb.GetSqlString(), sb.params...).Get(&stats)