Alerting: add state tracker to alerting evaluation (#32298)

* Initial commit for state tracking

* basic state transition logic and tests

* constructor. test and interface fixup

* use new sig for sch.definitionRoutine()

* test fixup

* make the linter happy

* more minor linting cleanup

pkg/services/ngalert/state/state_tracker.go

@@ -0,0 +1,100 @@
+package state
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/services/ngalert/eval"
+	"github.com/grafana/grafana/pkg/services/ngalert/models"
+)
+
+type AlertState struct {
+	UID     string
+	CacheId string
+	Labels  data.Labels
+	State   eval.State
+	Results []eval.State
+}
+
+type cache struct {
+	cacheMap map[string]AlertState
+	mu       sync.Mutex
+}
+
+type StateTracker struct {
+	stateCache cache
+}
+
+func NewStateTracker() *StateTracker {
+	return &StateTracker{
+		stateCache: cache{
+			cacheMap: make(map[string]AlertState),
+			mu:       sync.Mutex{},
+		},
+	}
+}
+
+func (c *cache) getOrCreate(uid string, result eval.Result) AlertState {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	idString := fmt.Sprintf("%s %s", uid, result.Instance.String())
+	if state, ok := c.cacheMap[idString]; ok {
+		return state
+	}
+	newState := AlertState{
+		UID:     uid,
+		CacheId: idString,
+		Labels:  result.Instance,
+		State:   result.State,
+		Results: []eval.State{result.State},
+	}
+	c.cacheMap[idString] = newState
+	return newState
+}
+
+func (c *cache) update(stateEntry AlertState) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.cacheMap[stateEntry.CacheId] = stateEntry
+}
+
+func (c *cache) getStateForEntry(stateId string) eval.State {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.cacheMap[stateId].State
+}
+
+func (st *StateTracker) ProcessEvalResults(uid string, results eval.Results, condition models.Condition) []AlertState {
+	var changedStates []AlertState
+	for _, result := range results {
+		currentState := st.stateCache.getOrCreate(uid, result)
+		currentState.Results = append(currentState.Results, result.State)
+		newState := st.getNextState(uid, result)
+		if newState != currentState.State {
+			currentState.State = newState
+			changedStates = append(changedStates, currentState)
+		}
+		st.stateCache.update(currentState)
+	}
+	return changedStates
+}
+
+func (st *StateTracker) getNextState(uid string, result eval.Result) eval.State {
+	currentState := st.stateCache.getOrCreate(uid, result)
+	if currentState.State == result.State {
+		return currentState.State
+	}
+
+	switch {
+	case currentState.State == result.State:
+		return currentState.State
+	case currentState.State == eval.Normal && result.State == eval.Alerting:
+		return eval.Alerting
+	case currentState.State == eval.Alerting && result.State == eval.Normal:
+		return eval.Normal
+	default:
+		return eval.Alerting
+	}
+}

pkg/services/ngalert/state/state_tracker_test.go

@@ -0,0 +1,123 @@
+package state
+
+import (
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/services/ngalert/eval"
+	"github.com/grafana/grafana/pkg/services/ngalert/models"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestProcessEvalResults(t *testing.T) {
+	testCases := []struct {
+		desc                 string
+		uid                  string
+		evalResults          eval.Results
+		condition            models.Condition
+		expectedCacheEntries int
+		expectedState        eval.State
+		expectedResultCount  int
+	}{
+		{
+			desc: "given a single evaluation result",
+			uid:  "test_uid",
+			evalResults: eval.Results{
+				eval.Result{
+					Instance: data.Labels{"label1": "value1", "label2": "value2"},
+				},
+			},
+			expectedCacheEntries: 1,
+			expectedState:        eval.Normal,
+			expectedResultCount:  0,
+		},
+		{
+			desc: "given a state change from normal to alerting",
+			uid:  "test_uid",
+			evalResults: eval.Results{
+				eval.Result{
+					Instance: data.Labels{"label1": "value1", "label2": "value2"},
+					State:    eval.Normal,
+				},
+				eval.Result{
+					Instance: data.Labels{"label1": "value1", "label2": "value2"},
+					State:    eval.Alerting,
+				},
+			},
+			expectedCacheEntries: 1,
+			expectedState:        eval.Alerting,
+			expectedResultCount:  1,
+		},
+		{
+			desc: "given a state change from alerting to normal",
+			uid:  "test_uid",
+			evalResults: eval.Results{
+				eval.Result{
+					Instance: data.Labels{"label1": "value1", "label2": "value2"},
+					State:    eval.Alerting,
+				},
+				eval.Result{
+					Instance: data.Labels{"label1": "value1", "label2": "value2"},
+					State:    eval.Normal,
+				},
+			},
+			expectedCacheEntries: 1,
+			expectedState:        eval.Normal,
+			expectedResultCount:  1,
+		},
+		{
+			desc: "given a constant alerting state",
+			uid:  "test_uid",
+			evalResults: eval.Results{
+				eval.Result{
+					Instance: data.Labels{"label1": "value1", "label2": "value2"},
+					State:    eval.Alerting,
+				},
+				eval.Result{
+					Instance: data.Labels{"label1": "value1", "label2": "value2"},
+					State:    eval.Alerting,
+				},
+			},
+			expectedCacheEntries: 1,
+			expectedState:        eval.Alerting,
+			expectedResultCount:  0,
+		},
+		{
+			desc: "given a constant normal state",
+			uid:  "test_uid",
+			evalResults: eval.Results{
+				eval.Result{
+					Instance: data.Labels{"label1": "value1", "label2": "value2"},
+					State:    eval.Normal,
+				},
+				eval.Result{
+					Instance: data.Labels{"label1": "value1", "label2": "value2"},
+					State:    eval.Normal,
+				},
+			},
+			expectedCacheEntries: 1,
+			expectedState:        eval.Normal,
+			expectedResultCount:  0,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run("the correct number of entries are added to the cache", func(t *testing.T) {
+			st := NewStateTracker()
+			st.ProcessEvalResults(tc.uid, tc.evalResults, tc.condition)
+			assert.Equal(t, len(st.stateCache.cacheMap), tc.expectedCacheEntries)
+		})
+
+		t.Run("the correct state is set", func(t *testing.T) {
+			st := NewStateTracker()
+			st.ProcessEvalResults(tc.uid, tc.evalResults, tc.condition)
+			assert.Equal(t, st.stateCache.getStateForEntry("test_uid label1=value1, label2=value2"), tc.expectedState)
+		})
+
+		t.Run("the correct number of results are returned", func(t *testing.T) {
+			st := NewStateTracker()
+			results := st.ProcessEvalResults(tc.uid, tc.evalResults, tc.condition)
+			assert.Equal(t, len(results), tc.expectedResultCount)
+		})
+	}
+}