From 0717ec11d6042bbc5b4cb8d908a396ecec78cdd5 Mon Sep 17 00:00:00 2001 From: Yuri Tseretyan Date: Tue, 15 Aug 2023 10:27:15 -0400 Subject: [PATCH] Alerting: Update state manager to change all current states in the case when Error\NoData is executed as Ok\Nomal (#68142) --- .../src/types/featureToggles.gen.ts | 1 + pkg/services/featuremgmt/registry.go | 8 + pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 + pkg/services/ngalert/eval/eval.go | 21 + pkg/services/ngalert/ngalert.go | 17 +- pkg/services/ngalert/state/manager.go | 81 +- .../ngalert/state/manager_private_test.go | 709 +++++++++++++++++- 8 files changed, 804 insertions(+), 38 deletions(-) diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index db38f708392..f4dc0f20828 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -119,4 +119,5 @@ export interface FeatureToggles { configurableSchedulerTick?: boolean; influxdbSqlSupport?: boolean; noBasicRole?: boolean; + alertingNoDataErrorExecution?: boolean; } diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 385bb76d8a0..9f96d219eb9 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -699,5 +699,13 @@ var ( Owner: grafanaAuthnzSquad, RequiresRestart: true, }, + { + Name: "alertingNoDataErrorExecution", + Description: "Changes how Alerting state manager handles execution of NoData/Error", + Stage: FeatureStagePrivatePreview, + FrontendOnly: false, + Owner: grafanaAlertingSquad, + RequiresRestart: true, + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 3c7d3c7f093..4c17618b286 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -100,3 +100,4 @@ prometheusConfigOverhaulAuth,experimental,@grafana/observability-metrics,false,f configurableSchedulerTick,experimental,@grafana/alerting-squad,false,false,true,false influxdbSqlSupport,experimental,@grafana/observability-metrics,false,false,false,false noBasicRole,experimental,@grafana/grafana-authnz-team,false,false,true,true +alertingNoDataErrorExecution,privatePreview,@grafana/alerting-squad,false,false,true,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index b7e3fcfb4e2..ead84f50bcd 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -410,4 +410,8 @@ const ( // FlagNoBasicRole // Enables a new role that has no permissions by default FlagNoBasicRole = "noBasicRole" + + // FlagAlertingNoDataErrorExecution + // Changes how Alerting state manager handles execution of NoData/Error + FlagAlertingNoDataErrorExecution = "alertingNoDataErrorExecution" ) diff --git a/pkg/services/ngalert/eval/eval.go b/pkg/services/ngalert/eval/eval.go index fd0e1ea6d91..e93e7a6d76c 100644 --- a/pkg/services/ngalert/eval/eval.go +++ b/pkg/services/ngalert/eval/eval.go @@ -142,6 +142,7 @@ type ExecutionResults struct { // Results is a slice of evaluated alert instances states. type Results []Result +// HasErrors returns true when Results contains at least one element with error func (evalResults Results) HasErrors() bool { for _, r := range evalResults { if r.State == Error { @@ -151,6 +152,26 @@ func (evalResults Results) HasErrors() bool { return false } +// HasErrors returns true when Results contains at least one element and all elements are errors +func (evalResults Results) IsError() bool { + for _, r := range evalResults { + if r.State != Error { + return false + } + } + return len(evalResults) > 0 +} + +// IsNoData returns true when all items are NoData or Results is empty +func (evalResults Results) IsNoData() bool { + for _, result := range evalResults { + if result.State != NoData { + return false + } + } + return true +} + // Result contains the evaluated State of an alert instance // identified by its labels. type Result struct { diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 37ef2f12e60..7a063df6980 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -212,14 +212,15 @@ func (ng *AlertNG) init() error { return err } cfg := state.ManagerCfg{ - Metrics: ng.Metrics.GetStateMetrics(), - ExternalURL: appUrl, - InstanceStore: ng.store, - Images: ng.ImageService, - Clock: clk, - Historian: history, - DoNotSaveNormalState: ng.FeatureToggles.IsEnabled(featuremgmt.FlagAlertingNoNormalState), - MaxStateSaveConcurrency: ng.Cfg.UnifiedAlerting.MaxStateSaveConcurrency, + Metrics: ng.Metrics.GetStateMetrics(), + ExternalURL: appUrl, + InstanceStore: ng.store, + Images: ng.ImageService, + Clock: clk, + Historian: history, + DoNotSaveNormalState: ng.FeatureToggles.IsEnabled(featuremgmt.FlagAlertingNoNormalState), + MaxStateSaveConcurrency: ng.Cfg.UnifiedAlerting.MaxStateSaveConcurrency, + ApplyNoDataAndErrorToAllStates: ng.FeatureToggles.IsEnabled(featuremgmt.FlagAlertingNoDataErrorExecution), } stateManager := state.NewManager(cfg) scheduler := schedule.NewScheduler(schedCfg, stateManager) diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index 9d3b8daf4fe..04eaa8e0bb5 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -40,8 +40,9 @@ type Manager struct { historian Historian externalURL *url.URL - doNotSaveNormalState bool - maxStateSaveConcurrency int + doNotSaveNormalState bool + maxStateSaveConcurrency int + applyNoDataAndErrorToAllStates bool } type ManagerCfg struct { @@ -55,25 +56,33 @@ type ManagerCfg struct { DoNotSaveNormalState bool // MaxStateSaveConcurrency controls the number of goroutines (per rule) that can save alert state in parallel. MaxStateSaveConcurrency int + + // ApplyNoDataAndErrorToAllStates makes state manager to apply exceptional results (NoData and Error) + // to all states when corresponding execution in the rule definition is set to either `Alerting` or `OK` + ApplyNoDataAndErrorToAllStates bool } func NewManager(cfg ManagerCfg) *Manager { return &Manager{ - cache: newCache(), - ResendDelay: ResendDelay, // TODO: make this configurable - log: log.New("ngalert.state.manager"), - metrics: cfg.Metrics, - instanceStore: cfg.InstanceStore, - images: cfg.Images, - historian: cfg.Historian, - clock: cfg.Clock, - externalURL: cfg.ExternalURL, - doNotSaveNormalState: cfg.DoNotSaveNormalState, - maxStateSaveConcurrency: cfg.MaxStateSaveConcurrency, + cache: newCache(), + ResendDelay: ResendDelay, // TODO: make this configurable + log: log.New("ngalert.state.manager"), + metrics: cfg.Metrics, + instanceStore: cfg.InstanceStore, + images: cfg.Images, + historian: cfg.Historian, + clock: cfg.Clock, + externalURL: cfg.ExternalURL, + doNotSaveNormalState: cfg.DoNotSaveNormalState, + maxStateSaveConcurrency: cfg.MaxStateSaveConcurrency, + applyNoDataAndErrorToAllStates: cfg.ApplyNoDataAndErrorToAllStates, } } func (st *Manager) Run(ctx context.Context) error { + if st.applyNoDataAndErrorToAllStates { + st.log.Info("Running in alternative execution of Error/NoData mode") + } ticker := st.clock.Ticker(MetricsScrapeInterval) for { select { @@ -244,12 +253,8 @@ func (st *Manager) ResetStateByRuleUID(ctx context.Context, rule *ngModels.Alert func (st *Manager) ProcessEvalResults(ctx context.Context, evaluatedAt time.Time, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels) []StateTransition { logger := st.log.FromContext(ctx) logger.Debug("State manager processing evaluation results", "resultCount", len(results)) - states := make([]StateTransition, 0, len(results)) + states := st.setNextStateForRule(ctx, alertRule, results, extraLabels, logger) - for _, result := range results { - s := st.setNextState(ctx, alertRule, result, extraLabels, logger) - states = append(states, s) - } staleStates := st.deleteStaleStatesFromCache(ctx, logger, evaluatedAt, alertRule) st.deleteAlertStates(ctx, logger, staleStates) @@ -262,10 +267,42 @@ func (st *Manager) ProcessEvalResults(ctx context.Context, evaluatedAt time.Time return allChanges } -// Set the current state based on evaluation results -func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRule, result eval.Result, extraLabels data.Labels, logger log.Logger) StateTransition { - currentState := st.cache.getOrCreate(ctx, logger, alertRule, result, extraLabels, st.externalURL) +func (st *Manager) setNextStateForRule(ctx context.Context, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels, logger log.Logger) []StateTransition { + if st.applyNoDataAndErrorToAllStates && results.IsNoData() && (alertRule.NoDataState == ngModels.Alerting || alertRule.NoDataState == ngModels.OK) { // If it is no data, check the mapping and switch all results to the new state + // TODO aggregate UID of datasources that returned NoData into one and provide as auxiliary info, probably annotation + transitions := st.setNextStateForAll(ctx, alertRule, results[0], logger) + if len(transitions) > 0 { + return transitions // if there are no current states for the rule. Create ones for each result + } + } + if st.applyNoDataAndErrorToAllStates && results.IsError() && (alertRule.ExecErrState == ngModels.AlertingErrState || alertRule.ExecErrState == ngModels.OkErrState) { + // TODO squash all errors into one, and provide as annotation + transitions := st.setNextStateForAll(ctx, alertRule, results[0], logger) + if len(transitions) > 0 { + return transitions // if there are no current states for the rule. Create ones for each result + } + } + transitions := make([]StateTransition, 0, len(results)) + for _, result := range results { + currentState := st.cache.getOrCreate(ctx, logger, alertRule, result, extraLabels, st.externalURL) + s := st.setNextState(ctx, alertRule, currentState, result, logger) + transitions = append(transitions, s) + } + return transitions +} +func (st *Manager) setNextStateForAll(ctx context.Context, alertRule *ngModels.AlertRule, result eval.Result, logger log.Logger) []StateTransition { + currentStates := st.cache.getStatesForRuleUID(alertRule.OrgID, alertRule.UID, false) + transitions := make([]StateTransition, 0, len(currentStates)) + for _, currentState := range currentStates { + t := st.setNextState(ctx, alertRule, currentState, result, logger) + transitions = append(transitions, t) + } + return transitions +} + +// Set the current state based on evaluation results +func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRule, currentState *State, result eval.Result, logger log.Logger) StateTransition { currentState.LastEvaluationTime = result.EvaluatedAt currentState.EvaluationDuration = result.EvaluationDuration currentState.Results = append(currentState.Results, Evaluation{ @@ -288,7 +325,7 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu // Usually, it happens in the case of classic conditions when the evalResult does not have labels. // // This is temporary change to make sure that the labels are not persistent in the state after it was in Error state - // TODO yuri. Remove it in https://github.com/grafana/grafana/pull/68142 + // TODO yuri. Remove it when correct Error result with labels is provided if currentState.State == eval.Error && result.State != eval.Error { // This is possible because state was updated after the CacheID was calculated. _, curOk := currentState.Labels["ref_id"] diff --git a/pkg/services/ngalert/state/manager_private_test.go b/pkg/services/ngalert/state/manager_private_test.go index ff03833545c..13f71d88609 100644 --- a/pkg/services/ngalert/state/manager_private_test.go +++ b/pkg/services/ngalert/state/manager_private_test.go @@ -194,6 +194,8 @@ func TestManager_saveAlertStates(t *testing.T) { // // t1[1:normal] t2[1:alerting] and 'for'=2 at t2 // t1[{}:alerting] t2[{}:normal] t3[NoData] at t2,t3 +// +//nolint:gocyclo func TestProcessEvalResults_StateTransitions(t *testing.T) { evaluationDuration := 10 * time.Millisecond evaluationInterval := 10 * time.Second @@ -304,7 +306,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { } } - executeTest := func(t *testing.T, alertRule *ngmodels.AlertRule, resultsAtTime map[time.Time]eval.Results, expectedTransitionsAtTime map[time.Time][]StateTransition) { + executeTest := func(t *testing.T, alertRule *ngmodels.AlertRule, resultsAtTime map[time.Time]eval.Results, expectedTransitionsAtTime map[time.Time][]StateTransition, applyNoDataErrorToAllStates bool) { clk := clock.NewMock() cfg := ManagerCfg{ @@ -315,6 +317,8 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { Clock: clk, Historian: &FakeHistorian{}, MaxStateSaveConcurrency: 1, + + ApplyNoDataAndErrorToAllStates: applyNoDataErrorToAllStates, } st := NewManager(cfg) @@ -856,7 +860,12 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - executeTest(t, tc.alertRule, tc.results, tc.expectedTransitions) + t.Run("applyNoDataErrorToAllStates=true", func(t *testing.T) { + executeTest(t, tc.alertRule, tc.results, tc.expectedTransitions, true) + }) + t.Run("applyNoDataErrorToAllStates=false", func(t *testing.T) { + executeTest(t, tc.alertRule, tc.results, tc.expectedTransitions, false) + }) }) } @@ -872,6 +881,8 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { ruleMutators []ngmodels.AlertRuleMutator results map[time.Time]eval.Results expectedTransitions map[ngmodels.NoDataState]map[time.Time][]StateTransition + + expectedTransitionsApplyNoDataErrorToAllStates map[ngmodels.NoDataState]map[time.Time][]StateTransition } executeForEachRule := func(t *testing.T, tc noDataTestCase) { @@ -885,11 +896,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { } } t.Run(fmt.Sprintf("execute as %s", stateExec), func(t *testing.T) { - expectedTransitions, ok := tc.expectedTransitions[stateExec] + expectedTransitions, ok := tc.expectedTransitionsApplyNoDataErrorToAllStates[stateExec] + overridden := "[*]" + if !ok { + expectedTransitions, ok = tc.expectedTransitions[stateExec] + overridden = "" + } if !ok { require.Fail(t, "no expected state transitions") } - executeTest(t, r, tc.results, expectedTransitions) + t.Run("applyNoDataErrorToAllStates=true"+overridden, func(t *testing.T) { + executeTest(t, r, tc.results, expectedTransitions, true) + }) + t.Run("applyNoDataErrorToAllStates=false", func(t *testing.T) { + expectedTransitions, ok := tc.expectedTransitions[stateExec] + if !ok { + require.Fail(t, "no expected state transitions") + } + executeTest(t, r, tc.results, expectedTransitions, false) + }) }) } } @@ -1023,9 +1048,49 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ + ngmodels.Alerting: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Alerting, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 3), + LastEvaluationTime: t2, + }, + }, + }, + }, + ngmodels.OK: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t2, + }, + }, + }, + }, + }, }, { - desc: "t1[1:normal,2:alerting] t2[NoData] t3[NoData] at t3", + desc: "t1[1:normal,2:alerting] t2[NoData] t3[NoData] at t2,t3", results: map[time.Time]eval.Results{ t1: { newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)), @@ -1040,6 +1105,21 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.NoData: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + no-data"], + State: eval.NoData, + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 3), + LastEvaluationTime: t2, + }, + }, + }, t3: { { PreviousState: eval.Normal, @@ -1185,6 +1265,149 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ + ngmodels.Alerting: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Alerting, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 3), + LastEvaluationTime: t2, + }, + }, + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Alerting, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t1, + EndsAt: t2.Add(ResendDelay * 3), + LastEvaluationTime: t2, + }, + }, + }, + t3: { + { + PreviousState: eval.Alerting, + PreviousStateReason: eval.NoData.String(), + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Alerting, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.NoData), + }, + StartsAt: t2, + EndsAt: t3.Add(ResendDelay * 3), + LastEvaluationTime: t3, + }, + }, + { + PreviousState: eval.Alerting, + PreviousStateReason: eval.NoData.String(), + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Alerting, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.NoData), + }, + StartsAt: t1, + EndsAt: t3.Add(ResendDelay * 3), + LastEvaluationTime: t3, + }, + }, + }, + }, + ngmodels.OK: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t2, + }, + }, + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Normal, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t2, + Resolved: true, + }, + }, + }, + t3: { + { + PreviousState: eval.Normal, + PreviousStateReason: eval.NoData.String(), + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.NoData), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t3, + }, + }, + { + PreviousState: eval.Normal, + PreviousStateReason: eval.NoData.String(), + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Normal, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t3, + }, + }, + }, + }, + }, }, { desc: "t1[1:normal,2:alerting] t2[NoData] t3[NoData] and 'for'=1 at t2*,t3", @@ -1203,6 +1426,21 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.NoData: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + no-data"], + State: eval.NoData, + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 3), + LastEvaluationTime: t2, + }, + }, + }, t3: { { PreviousState: eval.Normal, @@ -1358,6 +1596,136 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ + ngmodels.Alerting: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Pending, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 3), + LastEvaluationTime: t2, + }, + }, + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Alerting, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 3), + LastEvaluationTime: t2, + }, + }, + }, + t3: { + { + PreviousState: eval.Pending, + PreviousStateReason: eval.NoData.String(), + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Alerting, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t3, eval.NoData), + }, + StartsAt: t3, + EndsAt: t3.Add(ResendDelay * 3), + LastEvaluationTime: t3, + }, + }, + { + PreviousState: eval.Alerting, + PreviousStateReason: eval.NoData.String(), + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Alerting, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t3, eval.NoData), + }, + StartsAt: t2, + EndsAt: t3.Add(ResendDelay * 3), + LastEvaluationTime: t3, + }, + }, + }, + }, + ngmodels.OK: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t2, + }, + }, + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Normal, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t2, + }, + }, + }, + t3: { + { + PreviousState: eval.Normal, + PreviousStateReason: eval.NoData.String(), + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t3, eval.NoData), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t3, + }, + }, + { + PreviousState: eval.Normal, + PreviousStateReason: eval.NoData.String(), + State: &State{ + Labels: labels["system + rule + labels2"], + State: eval.Normal, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t3, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t3, + }, + }, + }, + }, + }, }, { desc: "t1[1:alerting] t2[NoData] t3[1:alerting] and 'for'=2 at t3", @@ -1429,6 +1797,46 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ + ngmodels.Alerting: { + t3: { + { + PreviousState: eval.Pending, + PreviousStateReason: eval.NoData.String(), + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Alerting, + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.Alerting), + }, + StartsAt: t3, + EndsAt: t3.Add(ResendDelay * 3), + LastEvaluationTime: t3, + }, + }, + }, + }, + ngmodels.OK: { + t3: { + { + PreviousState: eval.Normal, + PreviousStateReason: eval.NoData.String(), + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Pending, + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.Alerting), + }, + StartsAt: t3, + EndsAt: t3.Add(ResendDelay * 3), + LastEvaluationTime: t3, + }, + }, + }, + }, + }, }, { desc: "t1[NoData] t2[1:normal] t3[1:normal] at t3", @@ -1610,6 +2018,46 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ + ngmodels.Alerting: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule"], + State: eval.Alerting, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 3), + LastEvaluationTime: t2, + }, + }, + }, + }, + ngmodels.OK: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t2, + }, + }, + }, + }, + }, }, { desc: "t1[{}:alerting] t2[NoData] t3[NoData] at t3", @@ -1626,6 +2074,21 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.NoData: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + no-data"], + State: eval.NoData, + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 3), + LastEvaluationTime: t2, + }, + }, + }, t3: { { PreviousState: eval.Alerting, @@ -1729,9 +2192,88 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ + ngmodels.Alerting: { + t2: { + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule"], + State: eval.Alerting, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t1, + EndsAt: t2.Add(ResendDelay * 3), + LastEvaluationTime: t2, + }, + }, + }, + t3: { + { + PreviousState: eval.Alerting, + PreviousStateReason: eval.NoData.String(), + State: &State{ + Labels: labels["system + rule"], + State: eval.Alerting, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.NoData), + }, + StartsAt: t1, + EndsAt: t3.Add(ResendDelay * 3), + LastEvaluationTime: t3, + }, + }, + }, + }, + ngmodels.OK: { + t2: { + { + PreviousState: eval.Alerting, + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t2, + Resolved: true, + }, + }, + }, + t3: { + { + PreviousState: eval.Normal, + PreviousStateReason: eval.NoData.String(), + State: &State{ + Labels: labels["system + rule"], + State: eval.Normal, + StateReason: eval.NoData.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Alerting), + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t3, + }, + }, + }, + }, + }, }, { - desc: "t1[{}:alerting] t2[NoData] t3[{}:alerting] and 'for'=2 at t3", + desc: "t1[{}:alerting] t2[NoData] t3[{}:alerting] and 'for'=2 at t2*,t3", ruleMutators: []ngmodels.AlertRuleMutator{ngmodels.WithForNTimes(2)}, results: map[time.Time]eval.Results{ t1: { @@ -1746,6 +2288,21 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, expectedTransitions: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ ngmodels.NoData: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + no-data"], + State: eval.NoData, + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + }, + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 3), + LastEvaluationTime: t2, + }, + }, + }, t3: { { PreviousState: eval.Pending, @@ -1800,6 +2357,46 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.NoDataState]map[time.Time][]StateTransition{ + ngmodels.Alerting: { + t3: { + { + PreviousState: eval.Pending, + PreviousStateReason: eval.NoData.String(), + State: &State{ + Labels: labels["system + rule"], + State: eval.Alerting, + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.Alerting), + }, + StartsAt: t3, + EndsAt: t3.Add(ResendDelay * 3), + LastEvaluationTime: t3, + }, + }, + }, + }, + ngmodels.OK: { + t3: { + { + PreviousState: eval.Normal, + PreviousStateReason: eval.NoData.String(), + State: &State{ + Labels: labels["system + rule"], + State: eval.Pending, + Results: []Evaluation{ + newEvaluation(t2, eval.NoData), + newEvaluation(t3, eval.Alerting), + }, + StartsAt: t3, + EndsAt: t3.Add(ResendDelay * 3), + LastEvaluationTime: t3, + }, + }, + }, + }, + }, }, } @@ -1831,6 +2428,8 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { ruleMutators []ngmodels.AlertRuleMutator results map[time.Time]eval.Results expectedTransitions map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition + + expectedTransitionsApplyNoDataErrorToAllStates map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition } executeForEachRule := func(t *testing.T, tc errorTestCase) { @@ -1844,11 +2443,25 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { } } t.Run(fmt.Sprintf("execute as %s", stateExec), func(t *testing.T) { - expectedTransitions, ok := tc.expectedTransitions[stateExec] + expectedTransitions, ok := tc.expectedTransitionsApplyNoDataErrorToAllStates[stateExec] + overridden := "[*]" + if !ok { + expectedTransitions, ok = tc.expectedTransitions[stateExec] + overridden = "" + } if !ok { require.Fail(t, "no expected state transitions") } - executeTest(t, r, tc.results, expectedTransitions) + t.Run("applyNoDataErrorToAllStates=true"+overridden, func(t *testing.T) { + executeTest(t, r, tc.results, expectedTransitions, true) + }) + t.Run("applyNoDataErrorToAllStates=false", func(t *testing.T) { + expectedTransitions, ok := tc.expectedTransitions[stateExec] + if !ok { + require.Fail(t, "no expected state transitions") + } + executeTest(t, r, tc.results, expectedTransitions, false) + }) }) } } @@ -2063,6 +2676,45 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{ + ngmodels.AlertingErrState: { + t2: { + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Alerting, + StateReason: eval.Error.String(), + Error: datasourceError, + Results: []Evaluation{ + newEvaluation(t2, eval.Error), + }, + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 3), + LastEvaluationTime: t2, + }, + }, + }, + }, + ngmodels.OkErrState: { + t2: { + { + PreviousState: eval.Pending, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: eval.Error.String(), + Results: []Evaluation{ + newEvaluation(t2, eval.Error), + }, + StartsAt: t2, + EndsAt: t2, + LastEvaluationTime: t2, + }, + }, + }, + }, + }, }, { desc: "t1[1:normal] t2[QueryError] at t2", @@ -2135,6 +2787,47 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { }, }, }, + expectedTransitionsApplyNoDataErrorToAllStates: map[ngmodels.ExecutionErrorState]map[time.Time][]StateTransition{ + ngmodels.AlertingErrState: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Alerting, + StateReason: eval.Error.String(), + Error: datasourceError, + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.Error), + }, + StartsAt: t2, + EndsAt: t2.Add(ResendDelay * 3), + LastEvaluationTime: t2, + }, + }, + }, + }, + ngmodels.OkErrState: { + t2: { + { + PreviousState: eval.Normal, + State: &State{ + Labels: labels["system + rule + labels1"], + State: eval.Normal, + StateReason: eval.Error.String(), + Results: []Evaluation{ + newEvaluation(t1, eval.Normal), + newEvaluation(t2, eval.Error), + }, + StartsAt: t1, + EndsAt: t1, + LastEvaluationTime: t2, + }, + }, + }, + }, + }, }, { desc: "t1[QueryError] t2[1:normal] t3[1:normal] at t3",