Alerting: Update state manager to change all current states in the case when Error\NoData is executed as Ok\Nomal (#68142)

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -119,4 +119,5 @@ export interface FeatureToggles {
   configurableSchedulerTick?: boolean;
   influxdbSqlSupport?: boolean;
   noBasicRole?: boolean;
+  alertingNoDataErrorExecution?: boolean;
 }

pkg/services/featuremgmt/registry.go

@@ -699,5 +699,13 @@ var (
 			Owner:           grafanaAuthnzSquad,
 			RequiresRestart: true,
 		},
+		{
+			Name:            "alertingNoDataErrorExecution",
+			Description:     "Changes how Alerting state manager handles execution of NoData/Error",
+			Stage:           FeatureStagePrivatePreview,
+			FrontendOnly:    false,
+			Owner:           grafanaAlertingSquad,
+			RequiresRestart: true,
+		},
 	}
 )

pkg/services/featuremgmt/toggles_gen.csv

@@ -100,3 +100,4 @@ prometheusConfigOverhaulAuth,experimental,@grafana/observability-metrics,false,f
 configurableSchedulerTick,experimental,@grafana/alerting-squad,false,false,true,false
 influxdbSqlSupport,experimental,@grafana/observability-metrics,false,false,false,false
 noBasicRole,experimental,@grafana/grafana-authnz-team,false,false,true,true
+alertingNoDataErrorExecution,privatePreview,@grafana/alerting-squad,false,false,true,false

pkg/services/featuremgmt/toggles_gen.go

@@ -410,4 +410,8 @@ const (
 	// FlagNoBasicRole
 	// Enables a new role that has no permissions by default
 	FlagNoBasicRole = "noBasicRole"
+
+	// FlagAlertingNoDataErrorExecution
+	// Changes how Alerting state manager handles execution of NoData/Error
+	FlagAlertingNoDataErrorExecution = "alertingNoDataErrorExecution"
 )

pkg/services/ngalert/eval/eval.go

@@ -142,6 +142,7 @@ type ExecutionResults struct {
 // Results is a slice of evaluated alert instances states.
 type Results []Result
 
+// HasErrors returns true when Results contains at least one element with error
 func (evalResults Results) HasErrors() bool {
 	for _, r := range evalResults {
 		if r.State == Error {
@@ -151,6 +152,26 @@ func (evalResults Results) HasErrors() bool {
 	return false
 }
 
+// HasErrors returns true when Results contains at least one element and all elements are errors
+func (evalResults Results) IsError() bool {
+	for _, r := range evalResults {
+		if r.State != Error {
+			return false
+		}
+	}
+	return len(evalResults) > 0
+}
+
+// IsNoData returns true when all items are NoData or Results is empty
+func (evalResults Results) IsNoData() bool {
+	for _, result := range evalResults {
+		if result.State != NoData {
+			return false
+		}
+	}
+	return true
+}
+
 // Result contains the evaluated State of an alert instance
 // identified by its labels.
 type Result struct {

pkg/services/ngalert/ngalert.go

@@ -212,14 +212,15 @@ func (ng *AlertNG) init() error {
 		return err
 	}
 	cfg := state.ManagerCfg{
-		Metrics:                 ng.Metrics.GetStateMetrics(),
-		ExternalURL:             appUrl,
-		InstanceStore:           ng.store,
-		Images:                  ng.ImageService,
-		Clock:                   clk,
-		Historian:               history,
-		DoNotSaveNormalState:    ng.FeatureToggles.IsEnabled(featuremgmt.FlagAlertingNoNormalState),
-		MaxStateSaveConcurrency: ng.Cfg.UnifiedAlerting.MaxStateSaveConcurrency,
+		Metrics:                        ng.Metrics.GetStateMetrics(),
+		ExternalURL:                    appUrl,
+		InstanceStore:                  ng.store,
+		Images:                         ng.ImageService,
+		Clock:                          clk,
+		Historian:                      history,
+		DoNotSaveNormalState:           ng.FeatureToggles.IsEnabled(featuremgmt.FlagAlertingNoNormalState),
+		MaxStateSaveConcurrency:        ng.Cfg.UnifiedAlerting.MaxStateSaveConcurrency,
+		ApplyNoDataAndErrorToAllStates: ng.FeatureToggles.IsEnabled(featuremgmt.FlagAlertingNoDataErrorExecution),
 	}
 	stateManager := state.NewManager(cfg)
 	scheduler := schedule.NewScheduler(schedCfg, stateManager)

pkg/services/ngalert/state/manager.go

@@ -40,8 +40,9 @@ type Manager struct {
 	historian     Historian
 	externalURL   *url.URL
 
-	doNotSaveNormalState    bool
-	maxStateSaveConcurrency int
+	doNotSaveNormalState           bool
+	maxStateSaveConcurrency        int
+	applyNoDataAndErrorToAllStates bool
 }
 
 type ManagerCfg struct {
@@ -55,25 +56,33 @@ type ManagerCfg struct {
 	DoNotSaveNormalState bool
 	// MaxStateSaveConcurrency controls the number of goroutines (per rule) that can save alert state in parallel.
 	MaxStateSaveConcurrency int
+
+	// ApplyNoDataAndErrorToAllStates makes state manager to apply exceptional results (NoData and Error)
+	// to all states when corresponding execution in the rule definition is set to either `Alerting` or `OK`
+	ApplyNoDataAndErrorToAllStates bool
 }
 
 func NewManager(cfg ManagerCfg) *Manager {
 	return &Manager{
-		cache:                   newCache(),
-		ResendDelay:             ResendDelay, // TODO: make this configurable
-		log:                     log.New("ngalert.state.manager"),
-		metrics:                 cfg.Metrics,
-		instanceStore:           cfg.InstanceStore,
-		images:                  cfg.Images,
-		historian:               cfg.Historian,
-		clock:                   cfg.Clock,
-		externalURL:             cfg.ExternalURL,
-		doNotSaveNormalState:    cfg.DoNotSaveNormalState,
-		maxStateSaveConcurrency: cfg.MaxStateSaveConcurrency,
+		cache:                          newCache(),
+		ResendDelay:                    ResendDelay, // TODO: make this configurable
+		log:                            log.New("ngalert.state.manager"),
+		metrics:                        cfg.Metrics,
+		instanceStore:                  cfg.InstanceStore,
+		images:                         cfg.Images,
+		historian:                      cfg.Historian,
+		clock:                          cfg.Clock,
+		externalURL:                    cfg.ExternalURL,
+		doNotSaveNormalState:           cfg.DoNotSaveNormalState,
+		maxStateSaveConcurrency:        cfg.MaxStateSaveConcurrency,
+		applyNoDataAndErrorToAllStates: cfg.ApplyNoDataAndErrorToAllStates,
 	}
 }
 
 func (st *Manager) Run(ctx context.Context) error {
+	if st.applyNoDataAndErrorToAllStates {
+		st.log.Info("Running in alternative execution of Error/NoData mode")
+	}
 	ticker := st.clock.Ticker(MetricsScrapeInterval)
 	for {
 		select {
@@ -244,12 +253,8 @@ func (st *Manager) ResetStateByRuleUID(ctx context.Context, rule *ngModels.Alert
 func (st *Manager) ProcessEvalResults(ctx context.Context, evaluatedAt time.Time, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels) []StateTransition {
 	logger := st.log.FromContext(ctx)
 	logger.Debug("State manager processing evaluation results", "resultCount", len(results))
-	states := make([]StateTransition, 0, len(results))
+	states := st.setNextStateForRule(ctx, alertRule, results, extraLabels, logger)
 
-	for _, result := range results {
-		s := st.setNextState(ctx, alertRule, result, extraLabels, logger)
-		states = append(states, s)
-	}
 	staleStates := st.deleteStaleStatesFromCache(ctx, logger, evaluatedAt, alertRule)
 	st.deleteAlertStates(ctx, logger, staleStates)
 
@@ -262,10 +267,42 @@ func (st *Manager) ProcessEvalResults(ctx context.Context, evaluatedAt time.Time
 	return allChanges
 }
 
-// Set the current state based on evaluation results
-func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRule, result eval.Result, extraLabels data.Labels, logger log.Logger) StateTransition {
-	currentState := st.cache.getOrCreate(ctx, logger, alertRule, result, extraLabels, st.externalURL)
+func (st *Manager) setNextStateForRule(ctx context.Context, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels, logger log.Logger) []StateTransition {
+	if st.applyNoDataAndErrorToAllStates && results.IsNoData() && (alertRule.NoDataState == ngModels.Alerting || alertRule.NoDataState == ngModels.OK) { // If it is no data, check the mapping and switch all results to the new state
+		// TODO aggregate UID of datasources that returned NoData into one and provide as auxiliary info, probably annotation
+		transitions := st.setNextStateForAll(ctx, alertRule, results[0], logger)
+		if len(transitions) > 0 {
+			return transitions // if there are no current states for the rule. Create ones for each result
+		}
+	}
+	if st.applyNoDataAndErrorToAllStates && results.IsError() && (alertRule.ExecErrState == ngModels.AlertingErrState || alertRule.ExecErrState == ngModels.OkErrState) {
+		// TODO squash all errors into one, and provide as annotation
+		transitions := st.setNextStateForAll(ctx, alertRule, results[0], logger)
+		if len(transitions) > 0 {
+			return transitions // if there are no current states for the rule. Create ones for each result
+		}
+	}
+	transitions := make([]StateTransition, 0, len(results))
+	for _, result := range results {
+		currentState := st.cache.getOrCreate(ctx, logger, alertRule, result, extraLabels, st.externalURL)
+		s := st.setNextState(ctx, alertRule, currentState, result, logger)
+		transitions = append(transitions, s)
+	}
+	return transitions
+}
 
+func (st *Manager) setNextStateForAll(ctx context.Context, alertRule *ngModels.AlertRule, result eval.Result, logger log.Logger) []StateTransition {
+	currentStates := st.cache.getStatesForRuleUID(alertRule.OrgID, alertRule.UID, false)
+	transitions := make([]StateTransition, 0, len(currentStates))
+	for _, currentState := range currentStates {
+		t := st.setNextState(ctx, alertRule, currentState, result, logger)
+		transitions = append(transitions, t)
+	}
+	return transitions
+}
+
+// Set the current state based on evaluation results
+func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRule, currentState *State, result eval.Result, logger log.Logger) StateTransition {
 	currentState.LastEvaluationTime = result.EvaluatedAt
 	currentState.EvaluationDuration = result.EvaluationDuration
 	currentState.Results = append(currentState.Results, Evaluation{
@@ -288,7 +325,7 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu
 	// Usually, it happens in the case of classic conditions when the evalResult does not have labels.
 	//
 	// This is temporary change to make sure that the labels are not persistent in the state after it was in Error state
-	// TODO yuri. Remove it in https://github.com/grafana/grafana/pull/68142
+	// TODO yuri. Remove it when correct Error result with labels is provided
 	if currentState.State == eval.Error && result.State != eval.Error {
 		// This is possible because state was updated after the CacheID was calculated.
 		_, curOk := currentState.Labels["ref_id"]

pkg/services/ngalert/state/manager_private_test.go

@@ -194,6 +194,8 @@ func TestManager_saveAlertStates(t *testing.T) {
 //
 //	t1[1:normal] t2[1:alerting] and 'for'=2 at t2
 //	t1[{}:alerting] t2[{}:normal] t3[NoData] at t2,t3
+//
+//nolint:gocyclo
 func TestProcessEvalResults_StateTransitions(t *testing.T) {
 	evaluationDuration := 10 * time.Millisecond
 	evaluationInterval := 10 * time.Second
@@ -304,7 +306,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 		}
 	}
 
-	executeTest := func(t *testing.T, alertRule *ngmodels.AlertRule, resultsAtTime map[time.Time]eval.Results, expectedTransitionsAtTime map[time.Time][]StateTransition) {
+	executeTest := func(t *testing.T, alertRule *ngmodels.AlertRule, resultsAtTime map[time.Time]eval.Results, expectedTransitionsAtTime map[time.Time][]StateTransition, applyNoDataErrorToAllStates bool) {
 		clk := clock.NewMock()
 
 		cfg := ManagerCfg{
@@ -315,6 +317,8 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 			Clock:                   clk,
 			Historian:               &FakeHistorian{},
 			MaxStateSaveConcurrency: 1,
+
+			ApplyNoDataAndErrorToAllStates: applyNoDataErrorToAllStates,
 		}
 		st := NewManager(cfg)
 
@@ -856,7 +860,12 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.desc, func(t *testing.T) {
-			executeTest(t, tc.alertRule, tc.results, tc.expectedTransitions)
+			t.Run("applyNoDataErrorToAllStates=true", func(t *testing.T) {
+				executeTest(t, tc.alertRule, tc.results, tc.expectedTransitions, true)
+			})
+			t.Run("applyNoDataErrorToAllStates=false", func(t *testing.T) {
+				executeTest(t, tc.alertRule, tc.results, tc.expectedTransitions, false)
+			})
 		})
 	}
 
@@ -872,6 +881,8 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 			ruleMutators        []ngmodels.AlertRuleMutator
 			results             map[time.Time]eval.Results
 			expectedTransitions map[ngmodels.NoDataState]map[time.Time][]StateTransition
+
+			expectedTransitionsApplyNoDataErrorToAllStates map[ngmodels.NoDataState]map[time.Time][]StateTransition
 		}
 
 		executeForEachRule := func(t *testing.T, tc noDataTestCase) {
@@ -885,11 +896,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 					}
 				}
 				t.Run(fmt.Sprintf("execute as %s", stateExec), func(t *testing.T) {
-					expectedTransitions, ok := tc.expectedTransitions[stateExec]
+					expectedTransitions, ok := tc.expectedTransitionsApplyNoDataErrorToAllStates[stateExec]
+					overridden := "[*]"
+					if !ok {
+						expectedTransitions, ok = tc.expectedTransitions[stateExec]
+						overridden = ""
+					}
 					if !ok {
 						require.Fail(t, "no expected state transitions")
 					}
-					executeTest(t, r, tc.results, expectedTransitions)
+					t.Run("applyNoDataErrorToAllStates=true"+overridden, func(t *testing.T) {
+						executeTest(t, r, tc.results, expectedTransitions, true)
+					})
+					t.Run("applyNoDataErrorToAllStates=false", func(t *testing.T) {
+						expectedTransitions, ok := tc.expectedTransitions[stateExec]
+						if !ok {
+							require.Fail(t, "no expected state transitions")
+						}
+						executeTest(t, r, tc.results, expectedTransitions, false)
+					})
 				})
 			}
 		}
@@ -1023,9 +1048,49 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 						},
 					},
 				},
+				expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.Alerting: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1,
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+				},
 			},
 			{
-				desc: "t1[1:normal,2:alerting] t2[NoData] t3[NoData] at t3",
+				desc: "t1[1:normal,2:alerting] t2[NoData] t3[NoData] at t2,t3",
 				results: map[time.Time]eval.Results{
 					t1: {
 						newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
@@ -1040,6 +1105,21 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 				},
 				expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
 					ngmodels.NoData: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule + no-data"],
+									State:  eval.NoData,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
 						t3: {
 							{
 								PreviousState: eval.Normal,
@@ -1185,6 +1265,149 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 						},
 					},
 				},
+				expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.Alerting: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+							{
+								PreviousState: eval.Alerting,
+								State: &State{
+									Labels:      labels["system + rule + labels2"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+						t3: {
+							{
+								PreviousState:       eval.Alerting,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState:       eval.Alerting,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + labels2"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1,
+									LastEvaluationTime: t2,
+								},
+							},
+							{
+								PreviousState: eval.Alerting,
+								State: &State{
+									Labels:      labels["system + rule + labels2"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t2,
+									Resolved:           true,
+								},
+							},
+						},
+						t3: {
+							{
+								PreviousState:       eval.Normal,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState:       eval.Normal,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + labels2"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+				},
 			},
 			{
 				desc:         "t1[1:normal,2:alerting] t2[NoData] t3[NoData] and 'for'=1 at t2*,t3",
@@ -1203,6 +1426,21 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 				},
 				expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
 					ngmodels.NoData: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule + no-data"],
+									State:  eval.NoData,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
 						t3: {
 							{
 								PreviousState: eval.Normal,
@@ -1358,6 +1596,136 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 						},
 					},
 				},
+				expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.Alerting: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Pending,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels:      labels["system + rule + labels2"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+						t3: {
+							{
+								PreviousState:       eval.Pending,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState:       eval.Alerting,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + labels2"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1,
+									LastEvaluationTime: t2,
+								},
+							},
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels:      labels["system + rule + labels2"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+						t3: {
+							{
+								PreviousState:       eval.Normal,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1,
+									LastEvaluationTime: t3,
+								},
+							},
+							{
+								PreviousState:       eval.Normal,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule + labels2"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+				},
 			},
 			{
 				desc:         "t1[1:alerting] t2[NoData] t3[1:alerting] and 'for'=2 at t3",
@@ -1429,6 +1797,46 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 						},
 					},
 				},
+				expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.Alerting: {
+						t3: {
+							{
+								PreviousState:       eval.Pending,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels: labels["system + rule + labels1"],
+									State:  eval.Alerting,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.Alerting),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t3: {
+							{
+								PreviousState:       eval.Normal,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels: labels["system + rule + labels1"],
+									State:  eval.Pending,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.Alerting),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+				},
 			},
 			{
 				desc: "t1[NoData] t2[1:normal] t3[1:normal] at t3",
@@ -1610,6 +2018,46 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 						},
 					},
 				},
+				expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.Alerting: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1,
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+				},
 			},
 			{
 				desc: "t1[{}:alerting] t2[NoData] t3[NoData] at t3",
@@ -1626,6 +2074,21 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 				},
 				expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
 					ngmodels.NoData: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule + no-data"],
+									State:  eval.NoData,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
 						t3: {
 							{
 								PreviousState: eval.Alerting,
@@ -1729,9 +2192,88 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 						},
 					},
 				},
+				expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.Alerting: {
+						t2: {
+							{
+								PreviousState: eval.Alerting,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+						t3: {
+							{
+								PreviousState:       eval.Alerting,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Alerting,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t1,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t2: {
+							{
+								PreviousState: eval.Alerting,
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t2,
+									Resolved:           true,
+								},
+							},
+						},
+						t3: {
+							{
+								PreviousState:       eval.Normal,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels:      labels["system + rule"],
+									State:       eval.Normal,
+									StateReason: eval.NoData.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Alerting),
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+				},
 			},
 			{
-				desc:         "t1[{}:alerting] t2[NoData] t3[{}:alerting] and 'for'=2 at t3",
+				desc:         "t1[{}:alerting] t2[NoData] t3[{}:alerting] and 'for'=2 at t2*,t3",
 				ruleMutators: []ngmodels.AlertRuleMutator{ngmodels.WithForNTimes(2)},
 				results: map[time.Time]eval.Results{
 					t1: {
@@ -1746,6 +2288,21 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 				},
 				expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
 					ngmodels.NoData: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels: labels["system + rule + no-data"],
+									State:  eval.NoData,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
 						t3: {
 							{
 								PreviousState: eval.Pending,
@@ -1800,6 +2357,46 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 						},
 					},
 				},
+				expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{
+					ngmodels.Alerting: {
+						t3: {
+							{
+								PreviousState:       eval.Pending,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels: labels["system + rule"],
+									State:  eval.Alerting,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.Alerting),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+					ngmodels.OK: {
+						t3: {
+							{
+								PreviousState:       eval.Normal,
+								PreviousStateReason: eval.NoData.String(),
+								State: &State{
+									Labels: labels["system + rule"],
+									State:  eval.Pending,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.NoData),
+										newEvaluation(t3, eval.Alerting),
+									},
+									StartsAt:           t3,
+									EndsAt:             t3.Add(ResendDelay * 3),
+									LastEvaluationTime: t3,
+								},
+							},
+						},
+					},
+				},
 			},
 		}
 
@@ -1831,6 +2428,8 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 			ruleMutators        []ngmodels.AlertRuleMutator
 			results             map[time.Time]eval.Results
 			expectedTransitions map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition
+
+			expectedTransitionsApplyNoDataErrorToAllStates map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition
 		}
 
 		executeForEachRule := func(t *testing.T, tc errorTestCase) {
@@ -1844,11 +2443,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 					}
 				}
 				t.Run(fmt.Sprintf("execute as %s", stateExec), func(t *testing.T) {
-					expectedTransitions, ok := tc.expectedTransitions[stateExec]
+					expectedTransitions, ok := tc.expectedTransitionsApplyNoDataErrorToAllStates[stateExec]
+					overridden := "[*]"
+					if !ok {
+						expectedTransitions, ok = tc.expectedTransitions[stateExec]
+						overridden = ""
+					}
 					if !ok {
 						require.Fail(t, "no expected state transitions")
 					}
-					executeTest(t, r, tc.results, expectedTransitions)
+					t.Run("applyNoDataErrorToAllStates=true"+overridden, func(t *testing.T) {
+						executeTest(t, r, tc.results, expectedTransitions, true)
+					})
+					t.Run("applyNoDataErrorToAllStates=false", func(t *testing.T) {
+						expectedTransitions, ok := tc.expectedTransitions[stateExec]
+						if !ok {
+							require.Fail(t, "no expected state transitions")
+						}
+						executeTest(t, r, tc.results, expectedTransitions, false)
+					})
 				})
 			}
 		}
@@ -2063,6 +2676,45 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 						},
 					},
 				},
+				expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
+					ngmodels.AlertingErrState: {
+						t2: {
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Alerting,
+									StateReason: eval.Error.String(),
+									Error:       datasourceError,
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+					ngmodels.OkErrState: {
+						t2: {
+							{
+								PreviousState: eval.Pending,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Normal,
+									StateReason: eval.Error.String(),
+									Results: []Evaluation{
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2,
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+				},
 			},
 			{
 				desc: "t1[1:normal] t2[QueryError] at t2",
@@ -2135,6 +2787,47 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
 						},
 					},
 				},
+				expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{
+					ngmodels.AlertingErrState: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Alerting,
+									StateReason: eval.Error.String(),
+									Error:       datasourceError,
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t2,
+									EndsAt:             t2.Add(ResendDelay * 3),
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+					ngmodels.OkErrState: {
+						t2: {
+							{
+								PreviousState: eval.Normal,
+								State: &State{
+									Labels:      labels["system + rule + labels1"],
+									State:       eval.Normal,
+									StateReason: eval.Error.String(),
+									Results: []Evaluation{
+										newEvaluation(t1, eval.Normal),
+										newEvaluation(t2, eval.Error),
+									},
+									StartsAt:           t1,
+									EndsAt:             t1,
+									LastEvaluationTime: t2,
+								},
+							},
+						},
+					},
+				},
 			},
 			{
 				desc: "t1[QueryError] t2[1:normal] t3[1:normal] at t3",