From 9c05b30489c333cd8675fa1bd71a472fce7ad79f Mon Sep 17 00:00:00 2001
From: Yuri Tseretyan <yuriy.tseretyan@grafana.com>
Date: Mon, 15 Jul 2024 13:38:20 -0400
Subject: [PATCH] Chore: Add more logs and tracing to hysteresis flows (#90369)

---
 pkg/expr/hysteresis.go            | 20 +++++++++++++++-----
 pkg/services/ngalert/eval/eval.go |  5 +++--
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/pkg/expr/hysteresis.go b/pkg/expr/hysteresis.go
index 9f25653fca5..685af6ebaf1 100644
--- a/pkg/expr/hysteresis.go
+++ b/pkg/expr/hysteresis.go
@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"go.opentelemetry.io/otel/attribute"
 
 	"github.com/grafana/grafana/pkg/expr/mathexp"
 	"github.com/grafana/grafana/pkg/infra/tracing"
@@ -34,12 +35,18 @@ func (h *HysteresisCommand) NeedsVars() []string {
 func (h *HysteresisCommand) Execute(ctx context.Context, now time.Time, vars mathexp.Vars, tracer tracing.Tracer) (mathexp.Results, error) {
 	results := vars[h.ReferenceVar]
 
+	logger := logger.FromContext(ctx)
+	traceCtx, span := tracer.Start(ctx, "SSE.ExecuteHysteresis")
+	span.SetAttributes(attribute.Int("previousLoadedDimensions", len(h.LoadedDimensions)))
+	span.SetAttributes(attribute.Int("totalDimensions", len(results.Values)))
+	defer span.End()
+
 	// shortcut for NoData
 	if results.IsNoData() {
 		return mathexp.Results{Values: mathexp.Values{mathexp.NewNoData()}}, nil
 	}
 	if h.LoadedDimensions == nil || len(h.LoadedDimensions) == 0 {
-		return h.LoadingThresholdFunc.Execute(ctx, now, vars, tracer)
+		return h.LoadingThresholdFunc.Execute(traceCtx, now, vars, tracer)
 	}
 	var loadedVals, unloadedVals mathexp.Values
 	for _, value := range results.Values {
@@ -51,11 +58,14 @@ func (h *HysteresisCommand) Execute(ctx context.Context, now time.Time, vars mat
 		}
 	}
 
+	span.SetAttributes(attribute.Int("matchedLoadedDimensions", len(loadedVals)))
+
+	logger.Debug("Evaluating thresholds", "unloadingThresholdDimensions", len(loadedVals), "loadingThresholdDimensions", len(unloadedVals))
 	if len(loadedVals) == 0 { // if all values are unloaded
-		return h.LoadingThresholdFunc.Execute(ctx, now, vars, tracer)
+		return h.LoadingThresholdFunc.Execute(traceCtx, now, vars, tracer)
 	}
 	if len(unloadedVals) == 0 { // if all values are loaded
-		return h.UnloadingThresholdFunc.Execute(ctx, now, vars, tracer)
+		return h.UnloadingThresholdFunc.Execute(traceCtx, now, vars, tracer)
 	}
 
 	defer func() {
@@ -64,12 +74,12 @@ func (h *HysteresisCommand) Execute(ctx context.Context, now time.Time, vars mat
 	}()
 
 	vars[h.ReferenceVar] = mathexp.Results{Values: unloadedVals}
-	loadingResults, err := h.LoadingThresholdFunc.Execute(ctx, now, vars, tracer)
+	loadingResults, err := h.LoadingThresholdFunc.Execute(traceCtx, now, vars, tracer)
 	if err != nil {
 		return mathexp.Results{}, fmt.Errorf("failed to execute loading threshold: %w", err)
 	}
 	vars[h.ReferenceVar] = mathexp.Results{Values: loadedVals}
-	unloadingResults, err := h.UnloadingThresholdFunc.Execute(ctx, now, vars, tracer)
+	unloadingResults, err := h.UnloadingThresholdFunc.Execute(traceCtx, now, vars, tracer)
 	if err != nil {
 		return mathexp.Results{}, fmt.Errorf("failed to execute unloading threshold: %w", err)
 	}
diff --git a/pkg/services/ngalert/eval/eval.go b/pkg/services/ngalert/eval/eval.go
index 300113fa868..f01a0ea9ca3 100644
--- a/pkg/services/ngalert/eval/eval.go
+++ b/pkg/services/ngalert/eval/eval.go
@@ -387,8 +387,9 @@ func getExprRequest(ctx EvaluationContext, condition models.Condition, dsCacheSe
 					return nil, fmt.Errorf("recovery threshold '%s' is only allowed to be the alert condition", q.RefID)
 				}
 				if reader != nil {
-					logger.FromContext(ctx.Ctx).Debug("Detected hysteresis threshold command. Populating with the results")
-					err = q.PatchHysteresisExpression(reader.Read())
+					active := reader.Read()
+					logger.FromContext(ctx.Ctx).Debug("Detected hysteresis threshold command. Populating with the results", "items", len(active))
+					err = q.PatchHysteresisExpression(active)
 					if err != nil {
 						return nil, fmt.Errorf("failed to amend hysteresis command '%s': %w", q.RefID, err)
 					}