Instrumentation: Add histogram for request duration (#28364)

Signed-off-by: bergquist <carl.bergquist@gmail.com>

pkg/middleware/request_metrics.go

@@ -7,12 +7,14 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/infra/metrics"
+	"github.com/grafana/grafana/pkg/setting"
 	"github.com/prometheus/client_golang/prometheus"
 	"gopkg.in/macaron.v1"
 )
 
 var (
 	httpRequestsInFlight         prometheus.Gauge
+	httpRequestDurationHistogram *prometheus.HistogramVec
 )
 
 func init() {
@@ -23,11 +25,22 @@ func init() {
 		},
 	)
 
-	prometheus.MustRegister(httpRequestsInFlight)
+	httpRequestDurationHistogram = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: "grafana",
+			Name:      "http_request_duration_seconds",
+			Help:      "Histogram of latencies for HTTP requests.",
+			Buckets:   []float64{.1, .2, .4, 1, 3, 8, 20, 60, 120},
+		},
+		[]string{"handler"},
+	)
+
+	prometheus.MustRegister(httpRequestsInFlight, httpRequestDurationHistogram)
 }
 
 // RequestMetrics is a middleware handler that instruments the request
-func RequestMetrics(handler string) macaron.Handler {
+func RequestMetrics(cfg *setting.Cfg) func(handler string) macaron.Handler {
+	return func(handler string) macaron.Handler {
 		return func(res http.ResponseWriter, req *http.Request, c *macaron.Context) {
 			rw := res.(macaron.ResponseWriter)
 			now := time.Now()
@@ -40,8 +53,15 @@ func RequestMetrics(handler string) macaron.Handler {
 			code := sanitizeCode(status)
 			method := sanitizeMethod(req.Method)
 			metrics.MHttpRequestTotal.WithLabelValues(handler, code, method).Inc()
+
 			duration := time.Since(now).Nanoseconds() / int64(time.Millisecond)
+
+			// enable histogram and disable summaries for http requests.
+			if cfg.IsHTTPRequestHistogramEnabled() {
+				httpRequestDurationHistogram.WithLabelValues(handler).Observe(float64(duration))
+			} else {
 				metrics.MHttpRequestSummary.WithLabelValues(handler, code, method).Observe(float64(duration))
+			}
 
 			switch {
 			case strings.HasPrefix(req.RequestURI, "/api/datasources/proxy"):
@@ -53,6 +73,7 @@ func RequestMetrics(handler string) macaron.Handler {
 			}
 		}
 	}
+}
 
 func countApiRequests(status int) {
 	switch status {

pkg/server/server.go

@@ -273,7 +273,7 @@ func (s *Server) buildServiceGraph(services []*registry.Descriptor) error {
 	objs := []interface{}{
 		bus.GetBus(),
 		s.cfg,
-		routing.NewRouteRegister(middleware.RequestMetrics, middleware.RequestTracing),
+		routing.NewRouteRegister(middleware.RequestMetrics(s.cfg), middleware.RequestTracing),
 		localcache.New(5*time.Minute, 10*time.Minute),
 		s,
 	}

pkg/setting/setting.go

@@ -344,6 +344,10 @@ func (c Cfg) IsDatabaseMetricsEnabled() bool {
 	return c.FeatureToggles["database_metrics"]
 }
 
+func (c Cfg) IsHTTPRequestHistogramEnabled() bool {
+	return c.FeatureToggles["http_request_histogram"]
+}
+
 type CommandLineArgs struct {
 	Config   string
 	HomePath string