From 5510a0d8d120f9f7365ea3e3ada0d4ce4185584d Mon Sep 17 00:00:00 2001
From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>
Date: Thu, 28 Sep 2023 17:25:09 +0200
Subject: [PATCH] Loki: Add instrumentation for
 `grafana_loki_plugin_parse_response_duration_seconds_bucket` (#75570)

* Add metric for loki parsing response duration

* Rename and add exemplars

* Fix lint
---
 pkg/tsdb/loki/api.go                          |  4 ++-
 .../loki/instrumentation/instrumentation.go   | 33 +++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)
 create mode 100644 pkg/tsdb/loki/instrumentation/instrumentation.go

diff --git a/pkg/tsdb/loki/api.go b/pkg/tsdb/loki/api.go
index ae4ab2d06ac..cead56093cc 100644
--- a/pkg/tsdb/loki/api.go
+++ b/pkg/tsdb/loki/api.go
@@ -20,6 +20,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/grafana/grafana/pkg/tsdb/loki/instrumentation"
 	"github.com/grafana/grafana/pkg/util/converter"
 )
 
@@ -206,10 +207,11 @@ func (api *LokiAPI) DataQuery(ctx context.Context, query lokiQuery, responseOpts
 	if res.Error != nil {
 		span.RecordError(res.Error)
 		span.SetStatus(codes.Error, err.Error())
+		instrumentation.UpdatePluginParsingResponseDurationSeconds(ctx, time.Since(start), "error")
 		api.log.Error("Error parsing response from loki", "error", res.Error, "metricDataplane", responseOpts.metricDataplane, "duration", time.Since(start), "stage", stageParseResponse)
 		return nil, res.Error
 	}
-
+	instrumentation.UpdatePluginParsingResponseDurationSeconds(ctx, time.Since(start), "ok")
 	api.log.Info("Response parsed from loki", "duration", time.Since(start), "metricDataplane", responseOpts.metricDataplane, "framesLength", len(res.Frames), "stage", stageParseResponse)
 
 	return res.Frames, nil
diff --git a/pkg/tsdb/loki/instrumentation/instrumentation.go b/pkg/tsdb/loki/instrumentation/instrumentation.go
new file mode 100644
index 00000000000..8fadd73bcdf
--- /dev/null
+++ b/pkg/tsdb/loki/instrumentation/instrumentation.go
@@ -0,0 +1,33 @@
+package instrumentation
+
+import (
+	"context"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	pluginParsingResponseDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "grafana",
+		Name:      "loki_plugin_parse_response_duration_seconds",
+		Help:      "Duration of Loki parsing the response in seconds",
+		Buckets:   []float64{.001, 0.0025, .005, .0075, .01, .02, .03, .04, .05, .075, .1, .25, .5, 1, 5, 10, 25},
+	}, []string{"status", "endpoint"})
+)
+
+const (
+	EndpointQueryData = "queryData"
+)
+
+func UpdatePluginParsingResponseDurationSeconds(ctx context.Context, duration time.Duration, status string) {
+	histogram := pluginParsingResponseDurationSeconds.WithLabelValues(status, EndpointQueryData)
+
+	if traceID := tracing.TraceIDFromContext(ctx, true); traceID != "" {
+		histogram.(prometheus.ExemplarObserver).ObserveWithExemplar(duration.Seconds(), prometheus.Labels{"traceID": traceID})
+	} else {
+		histogram.Observe(duration.Seconds())
+	}
+}