Alerting: Improve performance of matching captures (#71828)

This commit updates eval.go to improve the performance of matching
captures in the general case. In some cases we have reduced the
runtime of the function from 10s of minutes to a couple 100ms.
In the case where no capture matches the exact labels, we revert to
the current subset/superset match, but with a reduced search space
due to grouping captures.

pkg/services/ngalert/eval/eval.go

@@ -302,14 +302,20 @@ type NumberValueCapture struct {
 }
 
 func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.QueryDataResponse) ExecutionResults {
-	// eval captures for the '__value_string__' annotation and the Value property of the API response.
-	captures := make([]NumberValueCapture, 0, len(execResp.Responses))
-	captureVal := func(refID string, labels data.Labels, value *float64) {
-		captures = append(captures, NumberValueCapture{
+	// captures contains the values of all instant queries and expressions for each dimension
+	captures := make(map[string]map[data.Fingerprint]NumberValueCapture)
+	captureFn := func(refID string, labels data.Labels, value *float64) {
+		m := captures[refID]
+		if m == nil {
+			m = make(map[data.Fingerprint]NumberValueCapture)
+		}
+		fp := labels.Fingerprint()
+		m[fp] = NumberValueCapture{
 			Var:    refID,
 			Value:  value,
 			Labels: labels.Copy(),
-		})
+		}
+		captures[refID] = m
 	}
 
 	// datasourceUIDsForRefIDs is a short-lived lookup table of RefID to DatasourceUID
@@ -357,7 +363,7 @@ func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.Q
 			if frame.Fields[0].Len() == 1 {
 				v = frame.At(0, 0).(*float64) // type checked above
 			}
-			captureVal(frame.RefID, frame.Fields[0].Labels, v)
+			captureFn(frame.RefID, frame.Fields[0].Labels, v)
 		}
 
 		if refID == c.Condition {
@@ -379,13 +385,27 @@ func queryDataResponseToExecutionResults(c models.Condition, execResp *backend.Q
 
 		if len(frame.Fields) == 1 {
 			theseLabels := frame.Fields[0].Labels
-			for _, cap := range captures {
-				// matching labels are equal labels, or when one set of labels includes the labels of the other.
-				if theseLabels.Equals(cap.Labels) || theseLabels.Contains(cap.Labels) || cap.Labels.Contains(theseLabels) {
+			fp := theseLabels.Fingerprint()
+
+			for _, fps := range captures {
+				// First look for a capture whose labels are an exact match
+				if v, ok := fps[fp]; ok {
 					if frame.Meta.Custom == nil {
 						frame.Meta.Custom = []NumberValueCapture{}
 					}
-					frame.Meta.Custom = append(frame.Meta.Custom.([]NumberValueCapture), cap)
+					frame.Meta.Custom = append(frame.Meta.Custom.([]NumberValueCapture), v)
+				} else {
+					// If no exact match was found, look for captures whose labels are either subsets
+					// or supersets
+					for _, v := range fps {
+						// matching labels are equal labels, or when one set of labels includes the labels of the other.
+						if theseLabels.Equals(v.Labels) || theseLabels.Contains(v.Labels) || v.Labels.Contains(theseLabels) {
+							if frame.Meta.Custom == nil {
+								frame.Meta.Custom = []NumberValueCapture{}
+							}
+							frame.Meta.Custom = append(frame.Meta.Custom.([]NumberValueCapture), v)
+						}
+					}
 				}
 			}
 		}

pkg/services/ngalert/eval/eval_bench_test.go

@@ -0,0 +1,59 @@
+package eval
+
+import (
+	"context"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/expr"
+	"github.com/grafana/grafana/pkg/services/ngalert/models"
+	"github.com/grafana/grafana/pkg/util"
+)
+
+func BenchmarkEvaluate(b *testing.B) {
+	var dataResp backend.QueryDataResponse
+	seedDataResponse(&dataResp, 10000)
+	var evaluator ConditionEvaluator = &conditionEvaluator{
+		expressionService: &fakeExpressionService{
+			hook: func(ctx context.Context, now time.Time, pipeline expr.DataPipeline) (*backend.QueryDataResponse, error) {
+				return &dataResp, nil
+			},
+		},
+		condition: models.Condition{
+			Condition: "B",
+		},
+	}
+	for i := 0; i < b.N; i++ {
+		_, err := evaluator.Evaluate(context.Background(), time.Now())
+		if err != nil {
+			b.Fatalf("Unexpected error: %s", err)
+		}
+	}
+}
+
+func seedDataResponse(r *backend.QueryDataResponse, n int) {
+	resps := make(backend.Responses, n)
+	for i := 0; i < n; i++ {
+		labels := data.Labels{
+			"foo": strconv.Itoa(i),
+			"bar": strconv.Itoa(i + 1),
+		}
+		a, b := resps["A"], resps["B"]
+		a.Frames = append(a.Frames, &data.Frame{
+			Fields: data.Fields{
+				data.NewField("Time", labels, []time.Time{time.Now()}),
+				data.NewField("Value", labels, []*float64{util.Pointer(1.0)}),
+			},
+		})
+		b.Frames = append(b.Frames, &data.Frame{
+			Fields: data.Fields{
+				data.NewField("Value", labels, []*float64{util.Pointer(1.0)}),
+			},
+		})
+		resps["A"], resps["B"] = a, b
+	}
+	r.Responses = resps
+}