Loki: Add instrumentation for grafana_loki_plugin_parse_response_duration_seconds_bucket (#75570)

* Add metric for loki parsing response duration

* Rename and add exemplars

* Fix lint

pkg/tsdb/loki/api.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/grafana/grafana/pkg/tsdb/loki/instrumentation"
 	"github.com/grafana/grafana/pkg/util/converter"
 )
 
@@ -206,10 +207,11 @@ func (api *LokiAPI) DataQuery(ctx context.Context, query lokiQuery, responseOpts
 	if res.Error != nil {
 		span.RecordError(res.Error)
 		span.SetStatus(codes.Error, err.Error())
+		instrumentation.UpdatePluginParsingResponseDurationSeconds(ctx, time.Since(start), "error")
 		api.log.Error("Error parsing response from loki", "error", res.Error, "metricDataplane", responseOpts.metricDataplane, "duration", time.Since(start), "stage", stageParseResponse)
 		return nil, res.Error
 	}
-
+	instrumentation.UpdatePluginParsingResponseDurationSeconds(ctx, time.Since(start), "ok")
 	api.log.Info("Response parsed from loki", "duration", time.Since(start), "metricDataplane", responseOpts.metricDataplane, "framesLength", len(res.Frames), "stage", stageParseResponse)
 
 	return res.Frames, nil

pkg/tsdb/loki/instrumentation/instrumentation.go

@@ -0,0 +1,33 @@
+package instrumentation
+
+import (
+	"context"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	pluginParsingResponseDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "grafana",
+		Name:      "loki_plugin_parse_response_duration_seconds",
+		Help:      "Duration of Loki parsing the response in seconds",
+		Buckets:   []float64{.001, 0.0025, .005, .0075, .01, .02, .03, .04, .05, .075, .1, .25, .5, 1, 5, 10, 25},
+	}, []string{"status", "endpoint"})
+)
+
+const (
+	EndpointQueryData = "queryData"
+)
+
+func UpdatePluginParsingResponseDurationSeconds(ctx context.Context, duration time.Duration, status string) {
+	histogram := pluginParsingResponseDurationSeconds.WithLabelValues(status, EndpointQueryData)
+
+	if traceID := tracing.TraceIDFromContext(ctx, true); traceID != "" {
+		histogram.(prometheus.ExemplarObserver).ObserveWithExemplar(duration.Seconds(), prometheus.Labels{"traceID": traceID})
+	} else {
+		histogram.Observe(duration.Seconds())
+	}
+}