mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
feat(alerting): extract logic state updates and notifications
ref #6444
This commit is contained in:
parent
b88791135f
commit
f0b591b89b
@ -26,6 +26,7 @@ type EvalContext struct {
|
||||
ImagePublicUrl string
|
||||
ImageOnDiskPath string
|
||||
NoDataFound bool
|
||||
PrevAlertState m.AlertStateType
|
||||
|
||||
Ctx context.Context
|
||||
}
|
||||
@ -63,6 +64,36 @@ func (c *EvalContext) GetStateModel() *StateDescription {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *EvalContext) ShouldUpdateAlertState() bool {
|
||||
return c.Rule.State != c.PrevAlertState
|
||||
}
|
||||
|
||||
func (c *EvalContext) ShouldSendNotification() bool {
|
||||
if (c.PrevAlertState == m.AlertStatePending) && (c.Rule.State == m.AlertStateOK) {
|
||||
return false
|
||||
}
|
||||
|
||||
alertState := c.Rule.State
|
||||
|
||||
if c.NoDataFound {
|
||||
if c.Rule.NoDataState == m.NoDataKeepState {
|
||||
return false
|
||||
}
|
||||
|
||||
alertState = c.Rule.NoDataState.ToAlertState()
|
||||
}
|
||||
|
||||
if c.Error != nil {
|
||||
if c.Rule.ExecutionErrorState == m.NoDataKeepState {
|
||||
return false
|
||||
}
|
||||
|
||||
alertState = c.Rule.ExecutionErrorState.ToAlertState()
|
||||
}
|
||||
|
||||
return alertState != c.PrevAlertState
|
||||
}
|
||||
|
||||
func (a *EvalContext) GetDurationMs() float64 {
|
||||
return float64(a.EndTime.Nanosecond()-a.StartTime.Nanosecond()) / float64(1000000)
|
||||
}
|
||||
|
110
pkg/services/alerting/eval_context_test.go
Normal file
110
pkg/services/alerting/eval_context_test.go
Normal file
@ -0,0 +1,110 @@
|
||||
package alerting
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
)
|
||||
|
||||
func TestAlertingEvalContext(t *testing.T) {
|
||||
Convey("Eval context", t, func() {
|
||||
ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
|
||||
err := fmt.Errorf("Dummie error!")
|
||||
|
||||
Convey("Should update alert state", func() {
|
||||
|
||||
Convey("ok -> alerting", func() {
|
||||
ctx.PrevAlertState = models.AlertStateOK
|
||||
ctx.Rule.State = models.AlertStateAlerting
|
||||
|
||||
So(ctx.ShouldUpdateAlertState(), ShouldBeTrue)
|
||||
})
|
||||
|
||||
Convey("ok -> ok", func() {
|
||||
ctx.PrevAlertState = models.AlertStateOK
|
||||
ctx.Rule.State = models.AlertStateOK
|
||||
|
||||
So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
|
||||
})
|
||||
})
|
||||
|
||||
Convey("Should send notifications", func() {
|
||||
Convey("pending -> ok", func() {
|
||||
ctx.PrevAlertState = models.AlertStatePending
|
||||
ctx.Rule.State = models.AlertStateOK
|
||||
|
||||
So(ctx.ShouldSendNotification(), ShouldBeFalse)
|
||||
})
|
||||
|
||||
Convey("ok -> alerting", func() {
|
||||
ctx.PrevAlertState = models.AlertStateOK
|
||||
ctx.Rule.State = models.AlertStateAlerting
|
||||
|
||||
So(ctx.ShouldSendNotification(), ShouldBeTrue)
|
||||
})
|
||||
|
||||
Convey("alerting -> ok", func() {
|
||||
ctx.PrevAlertState = models.AlertStateAlerting
|
||||
ctx.Rule.State = models.AlertStateOK
|
||||
|
||||
So(ctx.ShouldSendNotification(), ShouldBeTrue)
|
||||
})
|
||||
|
||||
Convey("ok -> no_data(alerting)", func() {
|
||||
ctx.PrevAlertState = models.AlertStateOK
|
||||
ctx.Rule.NoDataState = models.NoDataSetAlerting
|
||||
ctx.Rule.State = models.AlertStateAlerting
|
||||
|
||||
So(ctx.ShouldSendNotification(), ShouldBeTrue)
|
||||
})
|
||||
|
||||
Convey("ok -> no_data(ok)", func() {
|
||||
ctx.PrevAlertState = models.AlertStateOK
|
||||
ctx.Rule.NoDataState = models.NoDataSetOK
|
||||
ctx.NoDataFound = true
|
||||
ctx.Rule.State = models.AlertStateNoData
|
||||
|
||||
So(ctx.ShouldSendNotification(), ShouldBeFalse)
|
||||
})
|
||||
|
||||
Convey("ok -> no_data(keep_last)", func() {
|
||||
ctx.PrevAlertState = models.AlertStateOK
|
||||
ctx.Rule.NoDataState = models.NoDataKeepState
|
||||
ctx.Rule.State = models.AlertStateNoData
|
||||
ctx.NoDataFound = true
|
||||
|
||||
So(ctx.ShouldSendNotification(), ShouldBeFalse)
|
||||
})
|
||||
|
||||
Convey("ok -> execution_error(alerting)", func() {
|
||||
ctx.PrevAlertState = models.AlertStateOK
|
||||
ctx.Rule.State = models.AlertStateExecError
|
||||
ctx.Rule.ExecutionErrorState = models.NoDataSetAlerting
|
||||
ctx.Error = err
|
||||
|
||||
So(ctx.ShouldSendNotification(), ShouldBeTrue)
|
||||
})
|
||||
|
||||
Convey("ok -> execution_error(ok)", func() {
|
||||
ctx.PrevAlertState = models.AlertStateOK
|
||||
ctx.Rule.State = models.AlertStateExecError
|
||||
ctx.Rule.ExecutionErrorState = models.NoDataSetOK
|
||||
ctx.Error = err
|
||||
|
||||
So(ctx.ShouldSendNotification(), ShouldBeFalse)
|
||||
})
|
||||
|
||||
Convey("ok -> execution_error(keep_last)", func() {
|
||||
ctx.PrevAlertState = models.AlertStateOK
|
||||
ctx.Rule.State = models.AlertStateExecError
|
||||
ctx.Rule.ExecutionErrorState = models.NoDataKeepState
|
||||
ctx.Error = err
|
||||
|
||||
So(ctx.ShouldSendNotification(), ShouldBeFalse)
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
@ -28,7 +28,7 @@ func NewResultHandler() *DefaultResultHandler {
|
||||
}
|
||||
|
||||
func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
|
||||
oldState := evalContext.Rule.State
|
||||
evalContext.PrevAlertState = evalContext.Rule.State
|
||||
|
||||
executionError := ""
|
||||
annotationData := simplejson.New()
|
||||
@ -51,8 +51,8 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
|
||||
}
|
||||
|
||||
countStateResult(evalContext.Rule.State)
|
||||
if handler.shouldUpdateAlertState(evalContext, oldState) {
|
||||
handler.log.Info("New state change", "alertId", evalContext.Rule.Id, "newState", evalContext.Rule.State, "oldState", oldState)
|
||||
if evalContext.ShouldUpdateAlertState() {
|
||||
handler.log.Info("New state change", "alertId", evalContext.Rule.Id, "newState", evalContext.Rule.State, "prev state", evalContext.PrevAlertState)
|
||||
|
||||
cmd := &m.SetAlertStateCommand{
|
||||
AlertId: evalContext.Rule.Id,
|
||||
@ -76,7 +76,7 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
|
||||
Title: evalContext.Rule.Name,
|
||||
Text: evalContext.GetStateModel().Text,
|
||||
NewState: string(evalContext.Rule.State),
|
||||
PrevState: string(oldState),
|
||||
PrevState: string(evalContext.PrevAlertState),
|
||||
Epoch: time.Now().Unix(),
|
||||
Data: annotationData,
|
||||
}
|
||||
@ -86,21 +86,16 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
|
||||
handler.log.Error("Failed to save annotation for new alert state", "error", err)
|
||||
}
|
||||
|
||||
if (oldState == m.AlertStatePending) && (evalContext.Rule.State == m.AlertStateOK) {
|
||||
handler.log.Info("Notfication not sent", "oldState", oldState, "newState", evalContext.Rule.State)
|
||||
} else {
|
||||
if evalContext.ShouldSendNotification() {
|
||||
handler.notifier.Notify(evalContext)
|
||||
} else {
|
||||
handler.log.Info("Notfication not sent", "prev state", evalContext.PrevAlertState, "new state", evalContext.Rule.State)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (handler *DefaultResultHandler) shouldUpdateAlertState(evalContext *EvalContext, oldState m.AlertStateType) bool {
|
||||
return evalContext.Rule.State != oldState
|
||||
}
|
||||
|
||||
func countStateResult(state m.AlertStateType) {
|
||||
switch state {
|
||||
case m.AlertStatePending:
|
||||
|
@ -11,17 +11,18 @@ import (
|
||||
)
|
||||
|
||||
type Rule struct {
|
||||
Id int64
|
||||
OrgId int64
|
||||
DashboardId int64
|
||||
PanelId int64
|
||||
Frequency int64
|
||||
Name string
|
||||
Message string
|
||||
NoDataState m.NoDataOption
|
||||
State m.AlertStateType
|
||||
Conditions []Condition
|
||||
Notifications []int64
|
||||
Id int64
|
||||
OrgId int64
|
||||
DashboardId int64
|
||||
PanelId int64
|
||||
Frequency int64
|
||||
Name string
|
||||
Message string
|
||||
NoDataState m.NoDataOption
|
||||
ExecutionErrorState m.NoDataOption
|
||||
State m.AlertStateType
|
||||
Conditions []Condition
|
||||
Notifications []int64
|
||||
}
|
||||
|
||||
type ValidationError struct {
|
||||
|
Loading…
Reference in New Issue
Block a user