diff --git a/pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware.go b/pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware.go index b4ab647efed..3f8c95ac956 100644 --- a/pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware.go +++ b/pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware.go @@ -18,7 +18,7 @@ var ( Name: "datasource_request_total", Help: "A counter for outgoing requests for a data source", }, - []string{"datasource", "code", "method"}, + []string{"datasource", "datasource_type", "code", "method"}, ) datasourceRequestHistogram = promauto.NewHistogramVec( @@ -27,7 +27,7 @@ var ( Name: "datasource_request_duration_seconds", Help: "histogram of durations of outgoing data source requests sent from Grafana", Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25, 50, 100}, - }, []string{"datasource", "code", "method"}, + }, []string{"datasource", "datasource_type", "code", "method"}, ) datasourceResponseHistogram = promauto.NewHistogramVec( @@ -36,7 +36,7 @@ var ( Name: "datasource_response_size_bytes", Help: "histogram of data source response sizes returned to Grafana", Buckets: []float64{128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576}, - }, []string{"datasource"}, + }, []string{"datasource", "datasource_type"}, ) datasourceRequestsInFlight = promauto.NewGaugeVec( @@ -45,7 +45,7 @@ var ( Name: "datasource_request_in_flight", Help: "A gauge of outgoing data source requests currently being sent by Grafana", }, - []string{"datasource"}, + []string{"datasource", "datasource_type"}, ) ) @@ -71,18 +71,29 @@ func DataSourceMetricsMiddleware() sdkhttpclient.Middleware { return next } - datasourceLabel := prometheus.Labels{"datasource": datasourceLabelName} + datasourceType, exists := opts.Labels["datasource_type"] + if !exists { + return next + } + datasourceLabelType, err := metricutil.SanitizeLabelName(datasourceType) + // if the datasource type cannot be turned into a prometheus + // label we will skip instrumenting these metrics. + if err != nil { + return next + } - return executeMiddlewareFunc(next, datasourceLabel) + labels := prometheus.Labels{"datasource": datasourceLabelName, "datasource_type": datasourceLabelType} + + return executeMiddlewareFunc(next, labels) }) } -func executeMiddleware(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper { +func executeMiddleware(next http.RoundTripper, labels prometheus.Labels) http.RoundTripper { return sdkhttpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) { - requestCounter := datasourceRequestCounter.MustCurryWith(datasourceLabel) - requestHistogram := datasourceRequestHistogram.MustCurryWith(datasourceLabel) - requestInFlight := datasourceRequestsInFlight.With(datasourceLabel) - responseSizeHistogram := datasourceResponseHistogram.With(datasourceLabel) + requestCounter := datasourceRequestCounter.MustCurryWith(labels) + requestHistogram := datasourceRequestHistogram.MustCurryWith(labels) + requestInFlight := datasourceRequestsInFlight.With(labels) + responseSizeHistogram := datasourceResponseHistogram.With(labels) res, err := promhttp.InstrumentRoundTripperDuration(requestHistogram, promhttp.InstrumentRoundTripperCounter(requestCounter, diff --git a/pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware_test.go b/pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware_test.go index 905ad182cab..37afde4fd96 100644 --- a/pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware_test.go +++ b/pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware_test.go @@ -89,11 +89,11 @@ func TestDataSourceMetricsMiddleware(t *testing.T) { t.Run("With datasource name label options set should execute middleware", func(t *testing.T) { origExecuteMiddlewareFunc := executeMiddlewareFunc executeMiddlewareCalled := false - datasourceLabels := prometheus.Labels{} + labels := prometheus.Labels{} middlewareCalled := false executeMiddlewareFunc = func(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper { executeMiddlewareCalled = true - datasourceLabels = datasourceLabel + labels = datasourceLabel return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) { middlewareCalled = true return next.RoundTrip(r) @@ -106,7 +106,7 @@ func TestDataSourceMetricsMiddleware(t *testing.T) { ctx := &testContext{} finalRoundTripper := ctx.createRoundTripper("finalrt") mw := DataSourceMetricsMiddleware() - rt := mw.CreateMiddleware(httpclient.Options{Labels: map[string]string{"datasource_name": "My Data Source 123"}}, finalRoundTripper) + rt := mw.CreateMiddleware(httpclient.Options{Labels: map[string]string{"datasource_name": "My Data Source 123", "datasource_type": "prometheus"}}, finalRoundTripper) require.NotNil(t, rt) middlewareName, ok := mw.(httpclient.MiddlewareName) require.True(t, ok) @@ -123,8 +123,9 @@ func TestDataSourceMetricsMiddleware(t *testing.T) { require.Len(t, ctx.callChain, 1) require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain) require.True(t, executeMiddlewareCalled) - require.Len(t, datasourceLabels, 1) - require.Equal(t, "My_Data_Source_123", datasourceLabels["datasource"]) + require.Len(t, labels, 2) + require.Equal(t, "My_Data_Source_123", labels["datasource"]) + require.Equal(t, "prometheus", labels["datasource_type"]) require.True(t, middlewareCalled) }) }