Metrics: Add ability to disable classic histogram for HTTP metric (#88315)

metrics: Add ability to disable classic histogram for HTTP metric

Signed-off-by: Dave Henderson <dave.henderson@grafana.com>

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -151,6 +151,7 @@ Experimental features might be changed or removed without prior notice.
 | `pluginsAPIMetrics`                         | Sends metrics of public grafana packages usage by plugins                                                                                                                                                                                                                         |
 | `idForwarding`                              | Generate signed id token for identity that can be forwarded to plugins and external services                                                                                                                                                                                      |
 | `enableNativeHTTPHistogram`                 | Enables native HTTP Histograms                                                                                                                                                                                                                                                    |
+| `disableClassicHTTPHistogram`               | Disables classic HTTP Histogram (use with enableNativeHTTPHistogram)                                                                                                                                                                                                              |
 | `kubernetesSnapshots`                       | Routes snapshot requests from /api to the /apis endpoint                                                                                                                                                                                                                          |
 | `kubernetesDashboards`                      | Use the kubernetes API in the frontend for dashboards                                                                                                                                                                                                                             |
 | `datasourceQueryTypes`                      | Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus)                                                                                                                                                               |

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -112,6 +112,7 @@ export interface FeatureToggles {
   externalServiceAccounts?: boolean;
   panelMonitoring?: boolean;
   enableNativeHTTPHistogram?: boolean;
+  disableClassicHTTPHistogram?: boolean;
   formatString?: boolean;
   transformationsVariableSupport?: boolean;
   kubernetesPlaylists?: boolean;

pkg/middleware/request_metrics.go

@@ -51,12 +51,19 @@ func RequestMetrics(features featuremgmt.FeatureToggles, cfg *setting.Cfg, promR
 	if features.IsEnabledGlobally(featuremgmt.FlagEnableNativeHTTPHistogram) {
 		// the recommended default value from the prom_client
 		// https://github.com/prometheus/client_golang/blob/main/prometheus/histogram.go#L411
-		// Giving this variable an value means the client will expose the histograms as an
-		// native histogram instead of normal a normal histogram.
+		// Giving this variable a value means the client will expose a native
+		// histogram.
 		histogramOptions.NativeHistogramBucketFactor = 1.1
 		// The default value in OTel. It probably good enough for us as well.
 		histogramOptions.NativeHistogramMaxBucketNumber = 160
 		histogramOptions.NativeHistogramMinResetDuration = time.Hour
+
+		if features.IsEnabledGlobally(featuremgmt.FlagDisableClassicHTTPHistogram) {
+			// setting Buckets to nil with native options set means the classic
+			// histogram will no longer be exposed - this can be a good way to
+			// reduce cardinality in the exposed metrics
+			histogramOptions.Buckets = nil
+		}
 	}
 
 	httpRequestDurationHistogram := prometheus.NewHistogramVec(

pkg/services/featuremgmt/registry.go

@@ -700,11 +700,24 @@ var (
 			FrontendOnly: true,
 		},
 		{
-			Name:         "enableNativeHTTPHistogram",
-			Description:  "Enables native HTTP Histograms",
-			Stage:        FeatureStageExperimental,
-			FrontendOnly: false,
-			Owner:        hostedGrafanaTeam,
+			Name:              "enableNativeHTTPHistogram",
+			Description:       "Enables native HTTP Histograms",
+			Stage:             FeatureStageExperimental,
+			FrontendOnly:      false,
+			Owner:             grafanaBackendServicesSquad,
+			HideFromAdminPage: true,
+			AllowSelfServe:    false,
+			RequiresRestart:   true,
+		},
+		FeatureFlag{
+			Name:              "disableClassicHTTPHistogram",
+			Description:       "Disables classic HTTP Histogram (use with enableNativeHTTPHistogram)",
+			Stage:             FeatureStageExperimental,
+			FrontendOnly:      false,
+			Owner:             grafanaBackendServicesSquad,
+			HideFromAdminPage: true,
+			AllowSelfServe:    false,
+			RequiresRestart:   true,
 		},
 		{
 			Name:         "formatString",

pkg/services/featuremgmt/toggles_gen.csv

@@ -92,7 +92,8 @@ pluginsAPIMetrics,experimental,@grafana/plugins-platform-backend,false,false,tru
 idForwarding,experimental,@grafana/identity-access-team,false,false,false
 externalServiceAccounts,preview,@grafana/identity-access-team,false,false,false
 panelMonitoring,GA,@grafana/dataviz-squad,false,false,true
-enableNativeHTTPHistogram,experimental,@grafana/hosted-grafana-team,false,false,false
+enableNativeHTTPHistogram,experimental,@grafana/grafana-backend-services-squad,false,true,false
+disableClassicHTTPHistogram,experimental,@grafana/grafana-backend-services-squad,false,true,false
 formatString,preview,@grafana/dataviz-squad,false,false,true
 transformationsVariableSupport,GA,@grafana/dataviz-squad,false,false,true
 kubernetesPlaylists,GA,@grafana/grafana-app-platform-squad,false,true,false

pkg/services/featuremgmt/toggles_gen.go

@@ -383,6 +383,10 @@ const (
 	// Enables native HTTP Histograms
 	FlagEnableNativeHTTPHistogram = "enableNativeHTTPHistogram"
 
+	// FlagDisableClassicHTTPHistogram
+	// Disables classic HTTP Histogram (use with enableNativeHTTPHistogram)
+	FlagDisableClassicHTTPHistogram = "disableClassicHTTPHistogram"
+
 	// FlagFormatString
 	// Enable format string transformer
 	FlagFormatString = "formatString"