Instrumentation: Collect database connection stats (#52797)

Signed-off-by: bergquist <carl.bergquist@gmail.com>

pkg/services/sqlstore/sqlstore.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/go-sql-driver/mysql"
 	_ "github.com/lib/pq"
+	"github.com/prometheus/client_golang/prometheus"
 	"xorm.io/xorm"
 
 	"github.com/grafana/grafana/pkg/bus"
@@ -53,6 +54,17 @@ type SQLStore struct {
 	skipEnsureDefaultOrgAndUser bool
 	migrations                  registry.DatabaseMigrator
 	tracer                      tracing.Tracer
+	metrics                     struct {
+		maxOpenConnections prometheus.Gauge
+		openConnections    prometheus.Gauge
+		inUse              prometheus.Gauge
+		idle               prometheus.Gauge
+		waitCount          prometheus.Counter
+		waitDuration       prometheus.Counter
+		maxIdleClosed      prometheus.Counter
+		maxIdleTimeClosed  prometheus.Counter
+		maxLifetimeClosed  prometheus.Counter
+	}
 }
 
 func ProvideService(cfg *setting.Cfg, cacheService *localcache.CacheService, migrations registry.DatabaseMigrator, bus bus.Bus, tracer tracing.Tracer) (*SQLStore, error) {
@@ -73,6 +85,11 @@ func ProvideService(cfg *setting.Cfg, cacheService *localcache.CacheService, mig
 		return nil, err
 	}
 	s.tracer = tracer
+
+	s.initMetrics()
+
+	prometheus.MustRegister(s)
+
 	return s, nil
 }
 
@@ -425,6 +442,118 @@ func (ss *SQLStore) readConfig() error {
 	return nil
 }
 
+// initMetrics initializes the database connection metrics
+func (ss *SQLStore) initMetrics() {
+	namespace := "grafana"
+	subsystem := "database"
+
+	ss.metrics.maxOpenConnections = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      "conn_max_open",
+		Help:      "Maximum number of open connections to the database",
+	})
+
+	ss.metrics.openConnections = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      "conn_open",
+		Help:      "The number of established connections both in use and idle",
+	})
+
+	ss.metrics.inUse = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      "conn_in_use",
+		Help:      "The number of connections currently in use",
+	})
+
+	ss.metrics.idle = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      "conn_idle",
+		Help:      "The number of idle connections",
+	})
+
+	ss.metrics.waitCount = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      "conn_wait_count_total",
+		Help:      "The total number of connections waited for",
+	})
+
+	ss.metrics.waitDuration = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      "conn_wait_duration_seconds",
+		Help:      "The total time blocked waiting for a new connection",
+	})
+
+	ss.metrics.maxIdleClosed = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      "conn_max_idle_closed_total",
+		Help:      "The total number of connections closed due to SetMaxIdleConns",
+	})
+
+	ss.metrics.maxIdleTimeClosed = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      "conn_max_idle_closed_seconds",
+		Help:      "The total number of connections closed due to SetConnMaxIdleTime",
+	})
+
+	ss.metrics.maxLifetimeClosed = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      "conn_max_lifetime_closed_total",
+		Help:      "The total number of connections closed due to SetConnMaxLifetime",
+	})
+}
+
+// collectDBStats instruments connections stats from the database.
+func (ss *SQLStore) collectDBstats() {
+	dbstats := ss.engine.DB().Stats()
+	ss.metrics.maxOpenConnections.Set(float64(dbstats.MaxOpenConnections))
+	ss.metrics.openConnections.Set(float64(dbstats.MaxOpenConnections))
+	ss.metrics.inUse.Set(float64(dbstats.InUse))
+	ss.metrics.idle.Set(float64(dbstats.Idle))
+
+	ss.metrics.waitCount.Add(float64(dbstats.WaitCount))
+	ss.metrics.waitDuration.Add(float64(dbstats.WaitDuration / time.Second))
+	ss.metrics.maxIdleClosed.Add(float64(dbstats.MaxIdleClosed))
+	ss.metrics.maxIdleTimeClosed.Add(float64(dbstats.MaxIdleTimeClosed))
+	ss.metrics.maxLifetimeClosed.Add(float64(dbstats.MaxLifetimeClosed))
+}
+
+// Collect implements Prometheus.Collector.
+func (ss *SQLStore) Collect(ch chan<- prometheus.Metric) {
+	ss.collectDBstats()
+
+	ss.metrics.maxOpenConnections.Collect(ch)
+	ss.metrics.openConnections.Collect(ch)
+	ss.metrics.inUse.Collect(ch)
+	ss.metrics.idle.Collect(ch)
+	ss.metrics.waitCount.Collect(ch)
+	ss.metrics.waitDuration.Collect(ch)
+	ss.metrics.maxIdleClosed.Collect(ch)
+	ss.metrics.maxIdleTimeClosed.Collect(ch)
+	ss.metrics.maxLifetimeClosed.Collect(ch)
+}
+
+// Describe implements Prometheus.Collector.
+func (ss *SQLStore) Describe(ch chan<- *prometheus.Desc) {
+	ss.metrics.maxOpenConnections.Describe(ch)
+	ss.metrics.openConnections.Describe(ch)
+	ss.metrics.inUse.Describe(ch)
+	ss.metrics.idle.Describe(ch)
+	ss.metrics.waitCount.Describe(ch)
+	ss.metrics.waitDuration.Describe(ch)
+	ss.metrics.maxIdleClosed.Describe(ch)
+	ss.metrics.maxIdleTimeClosed.Describe(ch)
+	ss.metrics.maxLifetimeClosed.Describe(ch)
+}
+
 // ITestDB is an interface of arguments for testing db
 type ITestDB interface {
 	Helper()