SSE: Support hysteresis threshold expression (#70998)

* extend threshold command with second evaluator called `unloadEvaluator` 
* Introduce a new expression command Hysteresis and update Threshold unmarshaller to create the HysteresisCommand if the second eval
* add feature flag `recoveryThreshold`
* update unmarshal threshold command to not re-marshall because it breaks frame definition by shuffling the schema and data fields

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -148,6 +148,7 @@ Experimental features might be changed or removed without prior notice.
 | `transformationsVariableSupport`            | Allows using variables in transformations                                                                    |
 | `kubernetesPlaylists`                       | Use the kubernetes API in the frontend for playlists                                                         |
 | `navAdminSubsections`                       | Splits the administration section of the nav tree into subsections                                           |
+| `recoveryThreshold`                         | Enables feature recovery threshold (aka hysteresis) for threshold server-side expression                     |
 
 ## Development feature toggles
 

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -141,4 +141,5 @@ export interface FeatureToggles {
   transformationsVariableSupport?: boolean;
   kubernetesPlaylists?: boolean;
   navAdminSubsections?: boolean;
+  recoveryThreshold?: boolean;
 }

pkg/expr/graph.go

@@ -225,7 +225,7 @@ func (s *Service) buildGraph(req *Request) (*simple.DirectedGraph, error) {
 		case TypeDatasourceNode:
 			node, err = s.buildDSNode(dp, rn, req)
 		case TypeCMDNode:
-			node, err = buildCMDNode(dp, rn)
+			node, err = buildCMDNode(rn, s.features)
 		case TypeMLNode:
 			if s.features.IsEnabled(featuremgmt.FlagMlExpressions) {
 				node, err = s.buildMLNode(dp, rn, req)

pkg/expr/hysteresis.go

@@ -0,0 +1,122 @@
+package expr
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+
+	"github.com/grafana/grafana/pkg/expr/mathexp"
+	"github.com/grafana/grafana/pkg/infra/tracing"
+)
+
+type Fingerprints map[data.Fingerprint]struct{}
+
+// HysteresisCommand is a special case of ThresholdCommand that encapsulates two thresholds that are applied depending on the results of the previous evaluations:
+// - first threshold - "loading", is used when the metric is determined as not loaded, i.e. it does not exist in the data provided by the reader.
+// - second threshold - "unloading", is used when the metric is determined as loaded.
+// To determine whether a metric is loaded, the command uses LoadedDimensions that is supposed to contain data.Fingerprint of
+// the metrics that were loaded during the previous evaluation.
+// The result of the execution of the command is the same as ThresholdCommand: 0 or 1 for each metric.
+type HysteresisCommand struct {
+	RefID                  string
+	ReferenceVar           string
+	LoadingThresholdFunc   ThresholdCommand
+	UnloadingThresholdFunc ThresholdCommand
+	LoadedDimensions       Fingerprints
+}
+
+func (h *HysteresisCommand) NeedsVars() []string {
+	return []string{h.ReferenceVar}
+}
+
+func (h *HysteresisCommand) Execute(ctx context.Context, now time.Time, vars mathexp.Vars, tracer tracing.Tracer) (mathexp.Results, error) {
+	results := vars[h.ReferenceVar]
+
+	// shortcut for NoData
+	if results.IsNoData() {
+		return mathexp.Results{Values: mathexp.Values{mathexp.NewNoData()}}, nil
+	}
+	if h.LoadedDimensions == nil || len(h.LoadedDimensions) == 0 {
+		return h.LoadingThresholdFunc.Execute(ctx, now, vars, tracer)
+	}
+	var loadedVals, unloadedVals mathexp.Values
+	for _, value := range results.Values {
+		_, ok := h.LoadedDimensions[value.GetLabels().Fingerprint()]
+		if ok {
+			loadedVals = append(loadedVals, value)
+		} else {
+			unloadedVals = append(unloadedVals, value)
+		}
+	}
+
+	if len(loadedVals) == 0 { // if all values are unloaded
+		return h.LoadingThresholdFunc.Execute(ctx, now, vars, tracer)
+	}
+	if len(unloadedVals) == 0 { // if all values are loaded
+		return h.UnloadingThresholdFunc.Execute(ctx, now, vars, tracer)
+	}
+
+	defer func() {
+		// return back the old values
+		vars[h.ReferenceVar] = results
+	}()
+
+	vars[h.ReferenceVar] = mathexp.Results{Values: unloadedVals}
+	loadingResults, err := h.LoadingThresholdFunc.Execute(ctx, now, vars, tracer)
+	if err != nil {
+		return mathexp.Results{}, fmt.Errorf("failed to execute loading threshold: %w", err)
+	}
+	vars[h.ReferenceVar] = mathexp.Results{Values: loadedVals}
+	unloadingResults, err := h.UnloadingThresholdFunc.Execute(ctx, now, vars, tracer)
+	if err != nil {
+		return mathexp.Results{}, fmt.Errorf("failed to execute unloading threshold: %w", err)
+	}
+
+	return mathexp.Results{Values: append(loadingResults.Values, unloadingResults.Values...)}, nil
+}
+
+func NewHysteresisCommand(refID string, referenceVar string, loadCondition ThresholdCommand, unloadCondition ThresholdCommand, l Fingerprints) (*HysteresisCommand, error) {
+	return &HysteresisCommand{
+		RefID:                  refID,
+		LoadingThresholdFunc:   loadCondition,
+		UnloadingThresholdFunc: unloadCondition,
+		ReferenceVar:           referenceVar,
+		LoadedDimensions:       l,
+	}, nil
+}
+
+// FingerprintsFromFrame converts data.Frame to Fingerprints.
+// The input data frame must have a single field of uint64 type.
+// Returns error if the input data frame has invalid format
+func FingerprintsFromFrame(frame *data.Frame) (Fingerprints, error) {
+	frameType, frameVersion := frame.TypeInfo("")
+	if frameType != "fingerprints" {
+		return nil, fmt.Errorf("invalid format of loaded dimensions frame: expected frame type 'fingerprints'")
+	}
+	if frameVersion.Greater(data.FrameTypeVersion{1, 0}) {
+		return nil, fmt.Errorf("invalid format of loaded dimensions frame: expected frame type 'fingerprints' of version 1.0 or lower")
+	}
+	if len(frame.Fields) != 1 {
+		return nil, fmt.Errorf("invalid format of loaded dimensions frame: expected a single field but got %d", len(frame.Fields))
+	}
+	fld := frame.Fields[0]
+	if fld.Type() != data.FieldTypeUint64 {
+		return nil, fmt.Errorf("invalid format of loaded dimensions frame: the field type must be uint64 but got %s", fld.Type().String())
+	}
+	result := make(Fingerprints, fld.Len())
+	for i := 0; i < fld.Len(); i++ {
+		val, ok := fld.ConcreteAt(i)
+		if !ok {
+			continue
+		}
+		switch v := val.(type) {
+		case uint64:
+			result[data.Fingerprint(v)] = struct{}{}
+		default:
+			return nil, fmt.Errorf("cannot read the value at index [%d], expected uint64 but got '%T'", i, val)
+		}
+	}
+	return result, nil
+}

pkg/expr/hysteresis_test.go

@@ -0,0 +1,188 @@
+package expr
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/expr/mathexp"
+	"github.com/grafana/grafana/pkg/infra/tracing"
+)
+
+func TestHysteresisExecute(t *testing.T) {
+	number := func(label string, value float64) mathexp.Number {
+		n := mathexp.NewNumber("A", data.Labels{"label": label})
+		n.SetValue(&value)
+		return n
+	}
+	fingerprint := func(label string) data.Fingerprint {
+		return data.Labels{"label": label}.Fingerprint()
+	}
+
+	tracer := tracing.InitializeTracerForTest()
+
+	var loadThreshold = 100.0
+	var unloadThreshold = 30.0
+
+	testCases := []struct {
+		name             string
+		loadedDimensions Fingerprints
+		input            mathexp.Values
+		expected         mathexp.Values
+		expectedError    error
+	}{
+		{
+			name:             "return NoData when no data",
+			loadedDimensions: Fingerprints{0: struct{}{}},
+			input:            mathexp.Values{mathexp.NewNoData()},
+			expected:         mathexp.Values{mathexp.NewNoData()},
+		},
+		{
+			name:             "use only loaded condition if no loaded metrics",
+			loadedDimensions: Fingerprints{},
+			input: mathexp.Values{
+				number("value1", loadThreshold+1),
+				number("value2", loadThreshold),
+				number("value3", loadThreshold-1),
+				number("value4", unloadThreshold+1),
+				number("value5", unloadThreshold),
+				number("value6", unloadThreshold-1),
+			},
+			expected: mathexp.Values{
+				number("value1", 1),
+				number("value2", 0),
+				number("value3", 0),
+				number("value4", 0),
+				number("value5", 0),
+				number("value6", 0),
+			},
+		},
+		{
+			name: "evaluate loaded metrics against unloaded threshold",
+			loadedDimensions: Fingerprints{
+				fingerprint("value4"): {},
+				fingerprint("value5"): {},
+				fingerprint("value6"): {},
+			},
+			input: mathexp.Values{
+				number("value1", loadThreshold+1),
+				number("value2", loadThreshold),
+				number("value3", loadThreshold-1),
+				number("value4", unloadThreshold+1),
+				number("value5", unloadThreshold),
+				number("value6", unloadThreshold-1),
+			},
+			expected: mathexp.Values{
+				number("value1", 1),
+				number("value2", 0),
+				number("value3", 0),
+				number("value4", 1),
+				number("value5", 0),
+				number("value6", 0),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			cmd := &HysteresisCommand{
+				RefID:        "B",
+				ReferenceVar: "A",
+				LoadingThresholdFunc: ThresholdCommand{
+					ReferenceVar:  "A",
+					RefID:         "B",
+					ThresholdFunc: ThresholdIsAbove,
+					Conditions:    []float64{loadThreshold},
+				},
+				UnloadingThresholdFunc: ThresholdCommand{
+					ReferenceVar:  "A",
+					RefID:         "B",
+					ThresholdFunc: ThresholdIsAbove,
+					Conditions:    []float64{unloadThreshold},
+				},
+				LoadedDimensions: tc.loadedDimensions,
+			}
+
+			result, err := cmd.Execute(context.Background(), time.Now(), mathexp.Vars{
+				"A": mathexp.Results{Values: tc.input},
+			}, tracer)
+			if tc.expectedError != nil {
+				require.ErrorIs(t, err, tc.expectedError)
+				return
+			}
+			require.NoError(t, err)
+			require.EqualValues(t, result.Values, tc.expected)
+		})
+	}
+}
+
+func TestLoadedDimensionsFromFrame(t *testing.T) {
+	correctType := &data.FrameMeta{Type: "fingerprints", TypeVersion: data.FrameTypeVersion{1, 0}}
+	testCases := []struct {
+		name          string
+		frame         *data.Frame
+		expected      Fingerprints
+		expectedError bool
+	}{
+		{
+			name:          "should fail if frame has wrong type",
+			frame:         data.NewFrame("test").SetMeta(&data.FrameMeta{Type: "test"}),
+			expectedError: true,
+		},
+		{
+			name:          "should fail if frame has unsupported version",
+			frame:         data.NewFrame("test").SetMeta(&data.FrameMeta{Type: "fingerprints", TypeVersion: data.FrameTypeVersion{1, 1}}),
+			expectedError: true,
+		},
+		{
+			name:          "should fail if frame has no fields",
+			frame:         data.NewFrame("test").SetMeta(correctType),
+			expectedError: true,
+		},
+		{
+			name: "should fail if frame has many fields",
+			frame: data.NewFrame("test",
+				data.NewField("fingerprints", nil, []uint64{}),
+				data.NewField("test", nil, []string{}),
+			).SetMeta(correctType),
+			expectedError: true,
+		},
+		{
+			name: "should fail if frame has field of a wrong type",
+			frame: data.NewFrame("test",
+				data.NewField("fingerprints", nil, []int64{}),
+			).SetMeta(correctType),
+			expectedError: true,
+		},
+		{
+			name: "should fail if frame has nullable uint64 field",
+			frame: data.NewFrame("test",
+				data.NewField("fingerprints", nil, []*uint64{}),
+			).SetMeta(correctType),
+			expectedError: true,
+		},
+		{
+			name: "should create LoadedMetrics",
+			frame: data.NewFrame("test",
+				data.NewField("fingerprints", nil, []uint64{1, 2, 3, 4, 5}),
+			).SetMeta(correctType),
+			expected: Fingerprints{1: {}, 2: {}, 3: {}, 4: {}, 5: {}},
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			result, err := FingerprintsFromFrame(testCase.frame)
+			if testCase.expectedError {
+				require.Error(t, err)
+			} else {
+				require.EqualValues(t, testCase.expected, result)
+				b, _ := json.Marshal(testCase.frame)
+				t.Log(string(b))
+			}
+		})
+	}
+}

pkg/expr/mathexp/types.go

@@ -13,6 +13,11 @@ type Results struct {
 	Error  error
 }
 
+// IsNoData checks whether the result contains NoData value
+func (r Results) IsNoData() bool {
+	return len(r.Values) == 0 || len(r.Values) == 1 && r.Values[0].Type() == parse.TypeNoData
+}
+
 // Values is a slice of Value interfaces
 type Values []Value
 

pkg/expr/nodes.go

@@ -96,7 +96,7 @@ func (gn *CMDNode) Execute(ctx context.Context, now time.Time, vars mathexp.Vars
 	return gn.Command.Execute(ctx, now, vars, s.tracer)
 }
 
-func buildCMDNode(dp *simple.DirectedGraph, rn *rawNode) (*CMDNode, error) {
+func buildCMDNode(rn *rawNode, toggles featuremgmt.FeatureToggles) (*CMDNode, error) {
 	commandType, err := rn.GetCommandType()
 	if err != nil {
 		return nil, fmt.Errorf("invalid command type in expression '%v': %w", rn.RefID, err)
@@ -120,7 +120,7 @@ func buildCMDNode(dp *simple.DirectedGraph, rn *rawNode) (*CMDNode, error) {
 	case TypeClassicConditions:
 		node.Command, err = classic.UnmarshalConditionsCmd(rn.Query, rn.RefID)
 	case TypeThreshold:
-		node.Command, err = UnmarshalThresholdCommand(rn)
+		node.Command, err = UnmarshalThresholdCommand(rn, toggles)
 	default:
 		return nil, fmt.Errorf("expression command type '%v' in expression '%v' not implemented", commandType, rn.RefID)
 	}

pkg/expr/threshold.go

@@ -7,8 +7,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+
 	"github.com/grafana/grafana/pkg/expr/mathexp"
 	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
 )
 
 type ThresholdCommand struct {
@@ -16,6 +19,7 @@ type ThresholdCommand struct {
 	RefID         string
 	ThresholdFunc string
 	Conditions    []float64
+	Invert        bool
 }
 
 const (
@@ -39,6 +43,8 @@ func NewThresholdCommand(refID, referenceVar, thresholdFunc string, conditions [
 		if len(conditions) < 1 {
 			return nil, fmt.Errorf("incorrect number of arguments: got %d but need 1", len(conditions))
 		}
+	default:
+		return nil, fmt.Errorf("expected threshold function to be one of [%s], got %s", strings.Join(supportedThresholdFuncs, ", "), thresholdFunc)
 	}
 
 	return &ThresholdCommand{
@@ -49,50 +55,48 @@ func NewThresholdCommand(refID, referenceVar, thresholdFunc string, conditions [
 	}, nil
 }
 
-type ThresholdConditionJSON struct {
-	Evaluator ConditionEvalJSON `json:"evaluator"`
-}
-
 type ConditionEvalJSON struct {
 	Params []float64 `json:"params"`
 	Type   string    `json:"type"` // e.g. "gt"
 }
 
 // UnmarshalResampleCommand creates a ResampleCMD from Grafana's frontend query.
-func UnmarshalThresholdCommand(rn *rawNode) (*ThresholdCommand, error) {
+func UnmarshalThresholdCommand(rn *rawNode, features featuremgmt.FeatureToggles) (Command, error) {
-	rawQuery := rn.Query
+	cmdConfig := ThresholdCommandConfig{}
-
+	if err := json.Unmarshal(rn.QueryRaw, &cmdConfig); err != nil {
-	rawExpression, ok := rawQuery["expression"]
+		return nil, fmt.Errorf("failed to parse the threshold command: %w", err)
-	if !ok {
+	}
+	if cmdConfig.Expression == "" {
 		return nil, fmt.Errorf("no variable specified to reference for refId %v", rn.RefID)
 	}
-	referenceVar, ok := rawExpression.(string)
+	referenceVar := cmdConfig.Expression
-	if !ok {
-		return nil, fmt.Errorf("expected threshold variable to be a string, got %T for refId %v", rawExpression, rn.RefID)
-	}
-
-	jsonFromM, err := json.Marshal(rawQuery["conditions"])
-	if err != nil {
-		return nil, fmt.Errorf("failed to remarshal threshold expression body: %w", err)
-	}
-	var conditions []ThresholdConditionJSON
-	if err = json.Unmarshal(jsonFromM, &conditions); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal remarshaled threshold expression body: %w", err)
-	}
-
-	for _, condition := range conditions {
-		if !IsSupportedThresholdFunc(condition.Evaluator.Type) {
-			return nil, fmt.Errorf("expected threshold function to be one of %s, got %s", strings.Join(supportedThresholdFuncs, ", "), condition.Evaluator.Type)
-		}
-	}
 
 	// we only support one condition for now, we might want to turn this in to "OR" expressions later
-	if len(conditions) != 1 {
+	if len(cmdConfig.Conditions) != 1 {
 		return nil, fmt.Errorf("threshold expression requires exactly one condition")
 	}
-	firstCondition := conditions[0]
+	firstCondition := cmdConfig.Conditions[0]
 
-	return NewThresholdCommand(rn.RefID, referenceVar, firstCondition.Evaluator.Type, firstCondition.Evaluator.Params)
+	threshold, err := NewThresholdCommand(rn.RefID, referenceVar, firstCondition.Evaluator.Type, firstCondition.Evaluator.Params)
+	if err != nil {
+		return nil, fmt.Errorf("invalid condition: %w", err)
+	}
+	if firstCondition.UnloadEvaluator != nil && features.IsEnabled(featuremgmt.FlagRecoveryThreshold) {
+		unloading, err := NewThresholdCommand(rn.RefID, referenceVar, firstCondition.UnloadEvaluator.Type, firstCondition.UnloadEvaluator.Params)
+		unloading.Invert = true
+		if err != nil {
+			return nil, fmt.Errorf("invalid unloadCondition: %w", err)
+		}
+		var d Fingerprints
+		if firstCondition.LoadedDimensions != nil {
+			d, err = FingerprintsFromFrame(firstCondition.LoadedDimensions)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse loaded dimensions: %w", err)
+			}
+		}
+		return NewHysteresisCommand(rn.RefID, referenceVar, *threshold, *unloading, d)
+	}
+	return threshold, nil
 }
 
 // NeedsVars returns the variable names (refIds) that are dependencies
@@ -102,7 +106,7 @@ func (tc *ThresholdCommand) NeedsVars() []string {
 }
 
 func (tc *ThresholdCommand) Execute(ctx context.Context, now time.Time, vars mathexp.Vars, tracer tracing.Tracer) (mathexp.Results, error) {
-	mathExpression, err := createMathExpression(tc.ReferenceVar, tc.ThresholdFunc, tc.Conditions)
+	mathExpression, err := createMathExpression(tc.ReferenceVar, tc.ThresholdFunc, tc.Conditions, tc.Invert)
 	if err != nil {
 		return mathexp.Results{}, err
 	}
@@ -116,19 +120,25 @@ func (tc *ThresholdCommand) Execute(ctx context.Context, now time.Time, vars mat
 }
 
 // createMathExpression converts all the info we have about a "threshold" expression in to a Math expression
-func createMathExpression(referenceVar string, thresholdFunc string, args []float64) (string, error) {
+func createMathExpression(referenceVar string, thresholdFunc string, args []float64, invert bool) (string, error) {
+	var exp string
 	switch thresholdFunc {
 	case ThresholdIsAbove:
-		return fmt.Sprintf("${%s} > %f", referenceVar, args[0]), nil
+		exp = fmt.Sprintf("${%s} > %f", referenceVar, args[0])
 	case ThresholdIsBelow:
-		return fmt.Sprintf("${%s} < %f", referenceVar, args[0]), nil
+		exp = fmt.Sprintf("${%s} < %f", referenceVar, args[0])
 	case ThresholdIsWithinRange:
-		return fmt.Sprintf("${%s} > %f && ${%s} < %f", referenceVar, args[0], referenceVar, args[1]), nil
+		exp = fmt.Sprintf("${%s} > %f && ${%s} < %f", referenceVar, args[0], referenceVar, args[1])
 	case ThresholdIsOutsideRange:
-		return fmt.Sprintf("${%s} < %f || ${%s} > %f", referenceVar, args[0], referenceVar, args[1]), nil
+		exp = fmt.Sprintf("${%s} < %f || ${%s} > %f", referenceVar, args[0], referenceVar, args[1])
 	default:
 		return "", fmt.Errorf("failed to evaluate threshold expression: no such threshold function %s", thresholdFunc)
 	}
+
+	if invert {
+		return fmt.Sprintf("!(%s)", exp), nil
+	}
+	return exp, nil
 }
 
 func IsSupportedThresholdFunc(name string) bool {
@@ -142,3 +152,14 @@ func IsSupportedThresholdFunc(name string) bool {
 
 	return isSupported
 }
+
+type ThresholdCommandConfig struct {
+	Expression string                   `json:"expression"`
+	Conditions []ThresholdConditionJSON `json:"conditions"`
+}
+
+type ThresholdConditionJSON struct {
+	Evaluator        ConditionEvalJSON  `json:"evaluator"`
+	UnloadEvaluator  *ConditionEvalJSON `json:"unloadEvaluator"`
+	LoadedDimensions *data.Frame        `json:"loadedDimensions"`
+}

pkg/expr/threshold_test.go

@@ -2,9 +2,13 @@ package expr
 
 import (
 	"encoding/json"
+	"fmt"
+	"sort"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
 )
 
 func TestNewThresholdCommand(t *testing.T) {
@@ -82,6 +86,7 @@ func TestUnmarshalThresholdCommand(t *testing.T) {
 		query         string
 		shouldError   bool
 		expectedError string
+		assert        func(*testing.T, Command)
 	}
 
 	cases := []testCase{
@@ -97,7 +102,13 @@ func TestUnmarshalThresholdCommand(t *testing.T) {
 					}
 				}]
 			}`,
-			shouldError: false,
+			assert: func(t *testing.T, command Command) {
+				require.IsType(t, &ThresholdCommand{}, command)
+				cmd := command.(*ThresholdCommand)
+				require.Equal(t, []string{"A"}, cmd.NeedsVars())
+				require.Equal(t, "gt", cmd.ThresholdFunc)
+				require.Equal(t, []float64{20.0, 80.0}, cmd.Conditions)
+			},
 		},
 		{
 			description: "unmarshal with missing conditions should error",
@@ -107,17 +118,7 @@ func TestUnmarshalThresholdCommand(t *testing.T) {
 				"conditions": []
 			}`,
 			shouldError:   true,
-			expectedError: "requires exactly one condition",
+			expectedError: "threshold expression requires exactly one condition",
-		},
-		{
-			description: "unmarshal with missing conditions should error",
-			query: `{
-				"expression" : "A",
-				"type": "threshold",
-				"conditions": []
-			}`,
-			shouldError:   true,
-			expectedError: "requires exactly one condition",
 		},
 		{
 			description: "unmarshal with unsupported threshold function",
@@ -141,37 +142,86 @@ func TestUnmarshalThresholdCommand(t *testing.T) {
 				"type": "threshold",
 				"conditions": []
 			}`,
-			shouldError:   true,
+			shouldError: true,
-			expectedError: "expected threshold variable to be a string",
+		},
+		{
+			description: "unmarshal as hysteresis command if two evaluators",
+			query: `{
+				  "expression": "B",
+				  "conditions": [
+				    {
+				      "evaluator": {
+				        "params": [
+				          100
+				        ],
+				        "type": "gt"
+				      },
+				      "unloadEvaluator": {
+				        "params": [
+				          31
+				        ],
+				        "type": "lt"
+				      },
+				      "loadedDimensions": {"schema":{"name":"test","meta":{"type":"fingerprints","typeVersion":[1,0]},"fields":[{"name":"fingerprints","type":"number","typeInfo":{"frame":"uint64"}}]},"data":{"values":[[1,2,3,4,5]]}}
+				    }
+				  ]
+				}`,
+			assert: func(t *testing.T, c Command) {
+				require.IsType(t, &HysteresisCommand{}, c)
+				cmd := c.(*HysteresisCommand)
+				require.Equal(t, []string{"B"}, cmd.NeedsVars())
+				require.Equal(t, []string{"B"}, cmd.LoadingThresholdFunc.NeedsVars())
+				require.Equal(t, "gt", cmd.LoadingThresholdFunc.ThresholdFunc)
+				require.Equal(t, []float64{100.0}, cmd.LoadingThresholdFunc.Conditions)
+				require.Equal(t, []string{"B"}, cmd.UnloadingThresholdFunc.NeedsVars())
+				require.Equal(t, "lt", cmd.UnloadingThresholdFunc.ThresholdFunc)
+				require.Equal(t, []float64{31.0}, cmd.UnloadingThresholdFunc.Conditions)
+				require.True(t, cmd.UnloadingThresholdFunc.Invert)
+				require.NotNil(t, cmd.LoadedDimensions)
+				actual := make([]uint64, 0, len(cmd.LoadedDimensions))
+				for fingerprint := range cmd.LoadedDimensions {
+					actual = append(actual, uint64(fingerprint))
+				}
+				sort.Slice(actual, func(i, j int) bool {
+					return actual[i] < actual[j]
+				})
+
+				require.EqualValues(t, []uint64{1, 2, 3, 4, 5}, actual)
+			},
 		},
 	}
 
 	for _, tc := range cases {
-		q := []byte(tc.query)
+		t.Run(tc.description, func(t *testing.T) {
+			q := []byte(tc.query)
+			var qmap = make(map[string]any)
+			require.NoError(t, json.Unmarshal(q, &qmap))
 
-		var qmap = make(map[string]any)
+			cmd, err := UnmarshalThresholdCommand(&rawNode{
-		require.NoError(t, json.Unmarshal(q, &qmap))
+				RefID:      "",
+				Query:      qmap,
+				QueryRaw:   []byte(tc.query),
+				QueryType:  "",
+				DataSource: nil,
+			}, featuremgmt.WithFeatures(featuremgmt.FlagRecoveryThreshold))
 
-		cmd, err := UnmarshalThresholdCommand(&rawNode{
+			if tc.shouldError {
-			RefID:      "",
+				require.Nil(t, cmd)
-			Query:      qmap,
+				require.NotNil(t, err)
-			QueryType:  "",
+				require.Contains(t, err.Error(), tc.expectedError)
-			DataSource: nil,
+			} else {
+				require.Nil(t, err)
+				require.NotNil(t, cmd)
+				if tc.assert != nil {
+					tc.assert(t, cmd)
+				}
+			}
 		})
-
-		if tc.shouldError {
-			require.Nil(t, cmd)
-			require.NotNil(t, err)
-			require.Contains(t, err.Error(), tc.expectedError)
-		} else {
-			require.Nil(t, err)
-			require.NotNil(t, cmd)
-		}
 	}
 }
 
 func TestThresholdCommandVars(t *testing.T) {
-	cmd, err := NewThresholdCommand("B", "A", "is_above", []float64{})
+	cmd, err := NewThresholdCommand("B", "A", "lt", []float64{1.0})
 	require.Nil(t, err)
 	require.Equal(t, cmd.NeedsVars(), []string{"A"})
 }
@@ -219,17 +269,25 @@ func TestCreateMathExpression(t *testing.T) {
 
 	for _, tc := range cases {
 		t.Run(tc.description, func(t *testing.T) {
-			expr, err := createMathExpression(tc.ref, tc.function, tc.params)
+			expr, err := createMathExpression(tc.ref, tc.function, tc.params, false)
 
 			require.Nil(t, err)
 			require.NotNil(t, expr)
 
-			require.Equal(t, expr, tc.expected)
+			require.Equal(t, tc.expected, expr)
+
+			t.Run("inverted", func(t *testing.T) {
+				expr, err := createMathExpression(tc.ref, tc.function, tc.params, true)
+				require.Nil(t, err)
+				require.NotNil(t, expr)
+
+				require.Equal(t, fmt.Sprintf("!(%s)", tc.expected), expr)
+			})
 		})
 	}
 
 	t.Run("should error if function is unsupported", func(t *testing.T) {
-		expr, err := createMathExpression("A", "foo", []float64{0})
+		expr, err := createMathExpression("A", "foo", []float64{0}, false)
 		require.Equal(t, expr, "")
 		require.NotNil(t, err)
 		require.Contains(t, err.Error(), "no such threshold function")

pkg/services/featuremgmt/registry.go

@@ -859,5 +859,13 @@ var (
 			FrontendOnly: false,
 			Owner:        grafanaFrontendPlatformSquad,
 		},
+		{
+			Name:            "recoveryThreshold",
+			Description:     "Enables feature recovery threshold (aka hysteresis) for threshold server-side expression",
+			Stage:           FeatureStageExperimental,
+			FrontendOnly:    false,
+			Owner:           grafanaAlertingSquad,
+			RequiresRestart: true,
+		},
 	}
 )

pkg/services/featuremgmt/toggles_gen.csv

@@ -122,3 +122,4 @@ enableNativeHTTPHistogram,experimental,@grafana/hosted-grafana-team,false,false,
 transformationsVariableSupport,experimental,@grafana/grafana-bi-squad,false,false,false,true
 kubernetesPlaylists,experimental,@grafana/grafana-app-platform-squad,false,false,false,true
 navAdminSubsections,experimental,@grafana/grafana-frontend-platform,false,false,false,false
+recoveryThreshold,experimental,@grafana/alerting-squad,false,false,true,false

pkg/services/featuremgmt/toggles_gen.go

@@ -498,4 +498,8 @@ const (
 	// FlagNavAdminSubsections
 	// Splits the administration section of the nav tree into subsections
 	FlagNavAdminSubsections = "navAdminSubsections"
+
+	// FlagRecoveryThreshold
+	// Enables feature recovery threshold (aka hysteresis) for threshold server-side expression
+	FlagRecoveryThreshold = "recoveryThreshold"
 )