Instrumentation: Add histogram for request duration (#28364)

Signed-off-by: bergquist <carl.bergquist@gmail.com>

pkg/middleware/request_metrics.go

@@ -7,12 +7,14 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/infra/metrics"
+	"github.com/grafana/grafana/pkg/setting"
 	"github.com/prometheus/client_golang/prometheus"
 	"gopkg.in/macaron.v1"
 )
 
 var (
-	httpRequestsInFlight prometheus.Gauge
+	httpRequestsInFlight         prometheus.Gauge
+	httpRequestDurationHistogram *prometheus.HistogramVec
 )
 
 func init() {
@@ -23,33 +25,52 @@ func init() {
 		},
 	)
 
-	prometheus.MustRegister(httpRequestsInFlight)
+	httpRequestDurationHistogram = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: "grafana",
+			Name:      "http_request_duration_seconds",
+			Help:      "Histogram of latencies for HTTP requests.",
+			Buckets:   []float64{.1, .2, .4, 1, 3, 8, 20, 60, 120},
+		},
+		[]string{"handler"},
+	)
+
+	prometheus.MustRegister(httpRequestsInFlight, httpRequestDurationHistogram)
 }
 
 // RequestMetrics is a middleware handler that instruments the request
-func RequestMetrics(handler string) macaron.Handler {
-	return func(res http.ResponseWriter, req *http.Request, c *macaron.Context) {
-		rw := res.(macaron.ResponseWriter)
-		now := time.Now()
-		httpRequestsInFlight.Inc()
-		defer httpRequestsInFlight.Dec()
-		c.Next()
+func RequestMetrics(cfg *setting.Cfg) func(handler string) macaron.Handler {
+	return func(handler string) macaron.Handler {
+		return func(res http.ResponseWriter, req *http.Request, c *macaron.Context) {
+			rw := res.(macaron.ResponseWriter)
+			now := time.Now()
+			httpRequestsInFlight.Inc()
+			defer httpRequestsInFlight.Dec()
+			c.Next()
 
-		status := rw.Status()
+			status := rw.Status()
 
-		code := sanitizeCode(status)
-		method := sanitizeMethod(req.Method)
-		metrics.MHttpRequestTotal.WithLabelValues(handler, code, method).Inc()
-		duration := time.Since(now).Nanoseconds() / int64(time.Millisecond)
-		metrics.MHttpRequestSummary.WithLabelValues(handler, code, method).Observe(float64(duration))
+			code := sanitizeCode(status)
+			method := sanitizeMethod(req.Method)
+			metrics.MHttpRequestTotal.WithLabelValues(handler, code, method).Inc()
 
-		switch {
-		case strings.HasPrefix(req.RequestURI, "/api/datasources/proxy"):
-			countProxyRequests(status)
-		case strings.HasPrefix(req.RequestURI, "/api/"):
-			countApiRequests(status)
-		default:
-			countPageRequests(status)
+			duration := time.Since(now).Nanoseconds() / int64(time.Millisecond)
+
+			// enable histogram and disable summaries for http requests.
+			if cfg.IsHTTPRequestHistogramEnabled() {
+				httpRequestDurationHistogram.WithLabelValues(handler).Observe(float64(duration))
+			} else {
+				metrics.MHttpRequestSummary.WithLabelValues(handler, code, method).Observe(float64(duration))
+			}
+
+			switch {
+			case strings.HasPrefix(req.RequestURI, "/api/datasources/proxy"):
+				countProxyRequests(status)
+			case strings.HasPrefix(req.RequestURI, "/api/"):
+				countApiRequests(status)
+			default:
+				countPageRequests(status)
+			}
 		}
 	}
 }

pkg/server/server.go

@@ -273,7 +273,7 @@ func (s *Server) buildServiceGraph(services []*registry.Descriptor) error {
 	objs := []interface{}{
 		bus.GetBus(),
 		s.cfg,
-		routing.NewRouteRegister(middleware.RequestMetrics, middleware.RequestTracing),
+		routing.NewRouteRegister(middleware.RequestMetrics(s.cfg), middleware.RequestTracing),
 		localcache.New(5*time.Minute, 10*time.Minute),
 		s,
 	}

pkg/setting/setting.go

@@ -344,6 +344,10 @@ func (c Cfg) IsDatabaseMetricsEnabled() bool {
 	return c.FeatureToggles["database_metrics"]
 }
 
+func (c Cfg) IsHTTPRequestHistogramEnabled() bool {
+	return c.FeatureToggles["http_request_histogram"]
+}
+
 type CommandLineArgs struct {
 	Config   string
 	HomePath string