IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Additional Tests for State Manager (#41291)

Browse Source 
* rename fakeInstanceStore to FakeInstanceStore

* update test for state manager to initialize instance store with FakeInstanceStore
This commit is contained in:
Yuriy Tseretyan 2021-11-04 15:15:56 -04:00 committed by GitHub
parent e973b933f3
commit 1b5b747885
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 391 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
pkg/services/ngalert/schedule/schedule_unit_test.go
View File

@ -35,7 +35,7 @@ func TestSendingToExternalAlertmanager(t *testing.T) {
fakeAM := NewFakeExternalAlertmanager(t)
defer fakeAM.Close()
fakeRuleStore := newFakeRuleStore(t)
fakeInstanceStore := &fakeInstanceStore{}
fakeInstanceStore := &FakeInstanceStore{}
fakeAdminConfigStore := newFakeAdminConfigStore(t)
// create alert rule with one second interval
@ -100,7 +100,7 @@ func TestSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T) {
fakeAM := NewFakeExternalAlertmanager(t)
defer fakeAM.Close()
fakeRuleStore := newFakeRuleStore(t)
fakeInstanceStore := &fakeInstanceStore{}
fakeInstanceStore := &FakeInstanceStore{}
fakeAdminConfigStore := newFakeAdminConfigStore(t)
// First, let's create an admin configuration that holds an alertmanager.
@ -240,9 +240,9 @@ func TestSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T) {
func TestSchedule_ruleRoutine(t *testing.T) {
createSchedule := func(
evalAppliedChan chan time.Time,
) (*schedule, *fakeRuleStore, *fakeInstanceStore, *fakeAdminConfigStore, prometheus.Gatherer) {
) (*schedule, *fakeRuleStore, *FakeInstanceStore, *fakeAdminConfigStore, prometheus.Gatherer) {
ruleStore := newFakeRuleStore(t)
instanceStore := &fakeInstanceStore{}
instanceStore := &FakeInstanceStore{}
adminConfigStore := newFakeAdminConfigStore(t)
registry := prometheus.NewPedanticRegistry()

12
pkg/services/ngalert/schedule/testing.go
View File

@ -239,32 +239,32 @@ func (f *fakeRuleStore) UpdateRuleGroup(cmd store.UpdateRuleGroupCmd) error {
return nil
}
type fakeInstanceStore struct {
type FakeInstanceStore struct {
mtx sync.Mutex
recordedOps []interface{}
}
func (f *fakeInstanceStore) GetAlertInstance(q *models.GetAlertInstanceQuery) error {
func (f *FakeInstanceStore) GetAlertInstance(q *models.GetAlertInstanceQuery) error {
f.mtx.Lock()
defer f.mtx.Unlock()
f.recordedOps = append(f.recordedOps, *q)
return nil
}
func (f *fakeInstanceStore) ListAlertInstances(q *models.ListAlertInstancesQuery) error {
func (f *FakeInstanceStore) ListAlertInstances(q *models.ListAlertInstancesQuery) error {
f.mtx.Lock()
defer f.mtx.Unlock()
f.recordedOps = append(f.recordedOps, *q)
return nil
}
func (f *fakeInstanceStore) SaveAlertInstance(q *models.SaveAlertInstanceCommand) error {
func (f *FakeInstanceStore) SaveAlertInstance(q *models.SaveAlertInstanceCommand) error {
f.mtx.Lock()
defer f.mtx.Unlock()
f.recordedOps = append(f.recordedOps, *q)
return nil
}
func (f *fakeInstanceStore) FetchOrgIds() ([]int64, error) { return []int64{}, nil }
func (f *fakeInstanceStore) DeleteAlertInstance(_ int64, _, _ string) error { return nil }
func (f *FakeInstanceStore) FetchOrgIds() ([]int64, error) { return []int64{}, nil }
func (f *FakeInstanceStore) DeleteAlertInstance(_ int64, _, _ string) error { return nil }
func newFakeAdminConfigStore(t *testing.T) *fakeAdminConfigStore {
t.Helper()

382
pkg/services/ngalert/state/manager_test.go
View File

@ -6,10 +6,12 @@ import (
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/schedule"
"github.com/grafana/grafana/pkg/services/ngalert/state"
"github.com/grafana/grafana/pkg/services/ngalert/tests"
@ -356,6 +358,194 @@ func TestProcessEvalResults(t *testing.T) {
},
},
},
{
desc: "normal -> alerting -> noData -> alerting when For is set",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
For: 20 * time.Second,
NoDataState: models.NoData,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Alerting,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.NoData,
EvaluatedAt: evaluationTime.Add(20 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Alerting,
EvaluatedAt: evaluationTime.Add(30 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Alerting,
EvaluatedAt: evaluationTime.Add(40 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
},
expectedStates: map[string]*state.State{
`[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.Pending,
Results: []state.Evaluation{
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.Alerting,
Values: make(map[string]state.EvaluationValue),
},
{
EvaluationTime: evaluationTime.Add(20 * time.Second),
EvaluationState: eval.NoData,
Values: make(map[string]state.EvaluationValue),
},
{
EvaluationTime: evaluationTime.Add(30 * time.Second),
EvaluationState: eval.Alerting,
Values: make(map[string]state.EvaluationValue),
},
{
EvaluationTime: evaluationTime.Add(40 * time.Second),
EvaluationState: eval.Alerting,
Values: make(map[string]state.EvaluationValue),
},
},
StartsAt: evaluationTime.Add(30 * time.Second),
EndsAt: evaluationTime.Add(30 * time.Second).Add(state.ResendDelay * 3),
LastEvaluationTime: evaluationTime.Add(40 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "pending -> alerting -> noData when For is set and NoDataState is NoData",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
For: 20 * time.Second,
NoDataState: models.NoData,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Alerting,
EvaluatedAt: evaluationTime,
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Alerting,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Alerting,
EvaluatedAt: evaluationTime.Add(20 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.NoData,
EvaluatedAt: evaluationTime.Add(30 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
},
expectedStates: map[string]*state.State{
`[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.NoData,
Results: []state.Evaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Alerting,
Values: make(map[string]state.EvaluationValue),
},
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.Alerting,
Values: make(map[string]state.EvaluationValue),
},
{
EvaluationTime: evaluationTime.Add(20 * time.Second),
EvaluationState: eval.Alerting,
Values: make(map[string]state.EvaluationValue),
},
{
EvaluationTime: evaluationTime.Add(30 * time.Second),
EvaluationState: eval.NoData,
Values: make(map[string]state.EvaluationValue),
},
},
StartsAt: evaluationTime,
EndsAt: evaluationTime.Add(30 * time.Second).Add(state.ResendDelay * 3),
LastEvaluationTime: evaluationTime.Add(30 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "normal -> pending when For is set but not exceeded and first result is normal",
alertRule: &models.AlertRule{
@ -608,6 +798,192 @@ func TestProcessEvalResults(t *testing.T) {
},
},
},
{
desc: "normal -> nodata no labels when result is NoData and NoDataState is nodata",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
NoDataState: models.NoData,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{},
State: eval.NoData,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
},
expectedStates: map[string]*state.State{
`[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`,
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
},
State: eval.NoData,
Results: []state.Evaluation{
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.NoData,
Values: make(map[string]state.EvaluationValue),
},
},
StartsAt: evaluationTime.Add(10 * time.Second),
EndsAt: evaluationTime.Add(10 * time.Second).Add(state.ResendDelay * 3),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "normal (multi-dimensional) -> nodata no labels when result is NoData and NoDataState is nodata",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
NoDataState: models.NoData,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test-1"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
EvaluationDuration: evaluationDuration,
},
eval.Result{
Instance: data.Labels{"instance_label": "test-2"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{},
State: eval.NoData,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
},
expectedStates: map[string]*state.State{
`[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["label","test"]]`,
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
},
State: eval.NoData,
Results: []state.Evaluation{
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.NoData,
Values: make(map[string]state.EvaluationValue),
},
},
StartsAt: evaluationTime.Add(10 * time.Second),
EndsAt: evaluationTime.Add(10 * time.Second).Add(state.ResendDelay * 3),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "normal -> nodata no labels -> normal when result is NoData and NoDataState is nodata",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
NoDataState: models.NoData,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{},
State: eval.NoData,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime.Add(20 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
},
expectedStates: map[string]*state.State{
`[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.Normal,
Results: []state.Evaluation{
{
EvaluationTime: evaluationTime.Add(20 * time.Second),
EvaluationState: eval.Normal,
Values: make(map[string]state.EvaluationValue),
},
},
StartsAt: time.Time{},
EndsAt: time.Time{},
LastEvaluationTime: evaluationTime.Add(20 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "normal -> normal when result is NoData and NoDataState is ok",
alertRule: &models.AlertRule{
@ -852,11 +1228,15 @@ func TestProcessEvalResults(t *testing.T) {
}
for _, tc := range testCases {
st := state.NewManager(log.New("test_state_manager"), testMetrics.GetStateMetrics(), nil, nil, nil)
st := state.NewManager(log.New("test_state_manager"), testMetrics.GetStateMetrics(), nil, nil, &schedule.FakeInstanceStore{})
t.Run(tc.desc, func(t *testing.T) {
for _, res := range tc.evalResults {
_ = st.ProcessEvalResults(context.Background(), tc.alertRule, res)
}
states := st.GetStatesForRuleUID(tc.alertRule.OrgID, tc.alertRule.UID)
assert.Len(t, states, len(tc.expectedStates))
for _, s := range tc.expectedStates {
cachedState, err := st.Get(s.OrgID, s.AlertRuleUID, s.CacheId)
require.NoError(t, err)