Instrumentation: Enable native histograms for HTTP requests (#75731)

Signed-off-by: bergquist <carl.bergquist@gmail.com>

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -142,6 +142,7 @@ Experimental features might be changed or removed without prior notice.
 | `pluginsAPIMetrics`                         | Sends metrics of public grafana packages usage by plugins                                                    |
 | `httpSLOLevels`                             | Adds SLO level to http request metrics                                                                       |
 | `alertingModifiedExport`                    | Enables using UI for provisioned rules modification and export                                               |
+| `enableNativeHTTPHistogram`                 | Enables native HTTP Histograms                                                                               |
 
 ## Development feature toggles
 

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -135,4 +135,5 @@ export interface FeatureToggles {
   cloudWatchWildCardDimensionValues?: boolean;
   externalServiceAccounts?: boolean;
   alertingModifiedExport?: boolean;
+  enableNativeHTTPHistogram?: boolean;
 }

pkg/middleware/request_metrics.go

@@ -49,13 +49,26 @@ func RequestMetrics(features featuremgmt.FeatureToggles, cfg *setting.Cfg, promR
 		histogramLabels = append(histogramLabels, "slo_group")
 	}
 
+	histogramOptions := prometheus.HistogramOpts{
+		Namespace: "grafana",
+		Name:      "http_request_duration_seconds",
+		Help:      "Histogram of latencies for HTTP requests.",
+		Buckets:   defBuckets,
+	}
+
+	if features.IsEnabled(featuremgmt.FlagEnableNativeHTTPHistogram) {
+		// the recommended default value from the prom_client
+		// https://github.com/prometheus/client_golang/blob/main/prometheus/histogram.go#L411
+		// Giving this variable an value means the client will expose the histograms as an
+		// native histogram instead of normal a normal histogram.
+		histogramOptions.NativeHistogramBucketFactor = 1.1
+		// The default value in OTel. It probably good enough for us as well.
+		histogramOptions.NativeHistogramMaxBucketNumber = 160
+		histogramOptions.NativeHistogramMinResetDuration = time.Hour
+	}
+
 	httpRequestDurationHistogram := prometheus.NewHistogramVec(
-		prometheus.HistogramOpts{
-			Namespace: "grafana",
-			Name:      "http_request_duration_seconds",
-			Help:      "Histogram of latencies for HTTP requests.",
-			Buckets:   defBuckets,
-		},
+		histogramOptions,
 		histogramLabels,
 	)
 
@@ -108,16 +121,18 @@ func RequestMetrics(features featuremgmt.FeatureToggles, cfg *setting.Cfg, promR
 			histogram := httpRequestDurationHistogram.
 				WithLabelValues(labelValues...)
 
+			elapsedTime := time.Since(now).Seconds()
+
 			if traceID := tracing.TraceIDFromContext(r.Context(), true); traceID != "" {
 				// Need to type-convert the Observer to an
 				// ExemplarObserver. This will always work for a
 				// HistogramVec.
 				histogram.(prometheus.ExemplarObserver).ObserveWithExemplar(
-					time.Since(now).Seconds(), prometheus.Labels{"traceID": traceID},
+					elapsedTime, prometheus.Labels{"traceID": traceID},
 				)
-				return
+			} else {
+				histogram.Observe(elapsedTime)
 			}
-			histogram.Observe(time.Since(now).Seconds())
 
 			switch {
 			case strings.HasPrefix(r.RequestURI, "/api/datasources/proxy"):

pkg/services/featuremgmt/registry.go

@@ -816,5 +816,12 @@ var (
 			FrontendOnly: false,
 			Owner:        grafanaAlertingSquad,
 		},
+		{
+			Name:         "enableNativeHTTPHistogram",
+			Description:  "Enables native HTTP Histograms",
+			Stage:        FeatureStageExperimental,
+			FrontendOnly: false,
+			Owner:        hostedGrafanaTeam,
+		},
 	}
 )

pkg/services/featuremgmt/toggles_gen.csv

@@ -116,3 +116,4 @@ idForwarding,experimental,@grafana/grafana-authnz-team,true,false,false,false
 cloudWatchWildCardDimensionValues,GA,@grafana/aws-datasources,false,false,false,false
 externalServiceAccounts,experimental,@grafana/grafana-authnz-team,true,false,false,false
 alertingModifiedExport,experimental,@grafana/alerting-squad,false,false,false,false
+enableNativeHTTPHistogram,experimental,@grafana/hosted-grafana-team,false,false,false,false

pkg/services/featuremgmt/toggles_gen.go

@@ -474,4 +474,8 @@ const (
 	// FlagAlertingModifiedExport
 	// Enables using UI for provisioned rules modification and export
 	FlagAlertingModifiedExport = "alertingModifiedExport"
+
+	// FlagEnableNativeHTTPHistogram
+	// Enables native HTTP Histograms
+	FlagEnableNativeHTTPHistogram = "enableNativeHTTPHistogram"
 )