diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 937505f4c6b..4d20b13ad73 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -5,35 +5,32 @@ import ( "time" "github.com/benbjohnson/clock" - "github.com/grafana/grafana/pkg/bus" - "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/log" - m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting/alertstates" ) type Engine struct { - execQueue chan *AlertJob - resultQueue chan *AlertResult - clock clock.Clock - ticker *Ticker - scheduler Scheduler - handler AlertingHandler - ruleReader RuleReader - log log.Logger - notifier Notifier + execQueue chan *AlertJob + resultQueue chan *AlertResult + clock clock.Clock + ticker *Ticker + scheduler Scheduler + handler AlertingHandler + ruleReader RuleReader + log log.Logger + responseHandler ResultHandler } func NewEngine() *Engine { e := &Engine{ - ticker: NewTicker(time.Now(), time.Second*0, clock.New()), - execQueue: make(chan *AlertJob, 1000), - resultQueue: make(chan *AlertResult, 1000), - scheduler: NewScheduler(), - handler: NewHandler(), - ruleReader: NewRuleReader(), - log: log.New("alerting.engine"), - notifier: NewNotifier(), + ticker: NewTicker(time.Now(), time.Second*0, clock.New()), + execQueue: make(chan *AlertJob, 1000), + resultQueue: make(chan *AlertResult, 1000), + scheduler: NewScheduler(), + handler: NewHandler(), + ruleReader: NewRuleReader(), + log: log.New("alerting.engine"), + responseHandler: NewResultHandler(), } return e @@ -134,49 +131,13 @@ func (e *Engine) resultHandler() { result.State = alertstates.Critical result.Description = fmt.Sprintf("Failed to run check after %d retires, Error: %v", maxAlertExecutionRetries, result.Error) - e.reactToState(result) + //e.reactToState(result) + e.responseHandler.Handle(result) } } else { result.AlertJob.ResetRetry() - e.reactToState(result) + //e.reactToState(result) + e.responseHandler.Handle(result) } } } - -func (e *Engine) reactToState(result *AlertResult) { - if shouldUpdateState(result) { - cmd := &m.UpdateAlertStateCommand{ - AlertId: result.AlertJob.Rule.Id, - NewState: result.State, - Info: result.Description, - OrgId: result.AlertJob.Rule.OrgId, - TriggeredAlerts: simplejson.NewFromAny(result.TriggeredAlerts), - } - - if err := bus.Dispatch(cmd); err != nil { - e.log.Error("Failed to save state", "error", err) - } - - e.log.Debug("will notify about new state", "new state", result.State) - e.notifier.Notify(result) - } -} - -func shouldUpdateState(result *AlertResult) bool { - query := &m.GetLastAlertStateQuery{ - AlertId: result.AlertJob.Rule.Id, - OrgId: result.AlertJob.Rule.OrgId, - } - - if err := bus.Dispatch(query); err != nil { - log.Error2("Failed to read last alert state", "error", err) - return false - } - - now := time.Now() - noEarlierState := query.Result == nil - olderThen15Min := query.Result.Created.Before(now.Add(time.Minute * -15)) - changedState := query.Result.NewState != result.State - - return noEarlierState || changedState || olderThen15Min -} diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go new file mode 100644 index 00000000000..97ee346fa87 --- /dev/null +++ b/pkg/services/alerting/result_handler.go @@ -0,0 +1,68 @@ +package alerting + +import ( + "time" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" + m "github.com/grafana/grafana/pkg/models" +) + +type ResultHandler interface { + Handle(result *AlertResult) +} + +type ResultHandlerImpl struct { + notifier Notifier + log log.Logger +} + +func NewResultHandler() *ResultHandlerImpl { + return &ResultHandlerImpl{ + log: log.New("alerting.responseHandler"), + notifier: NewNotifier(), + } +} + +func (handler *ResultHandlerImpl) Handle(result *AlertResult) { + if handler.shouldUpdateState(result) { + cmd := &m.UpdateAlertStateCommand{ + AlertId: result.AlertJob.Rule.Id, + NewState: result.State, + Info: result.Description, + OrgId: result.AlertJob.Rule.OrgId, + TriggeredAlerts: simplejson.NewFromAny(result.TriggeredAlerts), + } + + if err := bus.Dispatch(cmd); err != nil { + handler.log.Error("Failed to save state", "error", err) + } + + handler.log.Debug("will notify about new state", "new state", result.State) + handler.notifier.Notify(result) + } +} + +func (handler *ResultHandlerImpl) shouldUpdateState(result *AlertResult) bool { + query := &m.GetLastAlertStateQuery{ + AlertId: result.AlertJob.Rule.Id, + OrgId: result.AlertJob.Rule.OrgId, + } + + if err := bus.Dispatch(query); err != nil { + log.Error2("Failed to read last alert state", "error", err) + return false + } + + now := time.Now() + + if query.Result == nil { + return true + } + + olderThen15Min := query.Result.Created.Before(now.Add(time.Minute * -15)) + changedState := query.Result.NewState != result.State + + return changedState || olderThen15Min +} diff --git a/pkg/services/alerting/result_handler_test.go b/pkg/services/alerting/result_handler_test.go new file mode 100644 index 00000000000..bd492259b84 --- /dev/null +++ b/pkg/services/alerting/result_handler_test.go @@ -0,0 +1,58 @@ +package alerting + +import ( + "testing" + "time" + + "github.com/grafana/grafana/pkg/bus" + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting/alertstates" + . "github.com/smartystreets/goconvey/convey" +) + +func TestAlertResultHandler(t *testing.T) { + Convey("Test result Handler", t, func() { + resultHandler := ResultHandlerImpl{} + mockResult := &AlertResult{ + State: alertstates.Ok, + AlertJob: &AlertJob{ + Rule: &AlertRule{ + Id: 1, + OrgId: 1, + }, + }, + } + mockAlertState := &m.AlertState{} + bus.ClearBusHandlers() + bus.AddHandler("test", func(query *m.GetLastAlertStateQuery) error { + query.Result = mockAlertState + return nil + }) + + Convey("Should update", func() { + + Convey("when no earlier alert state", func() { + mockAlertState = nil + So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue) + }) + + Convey("alert state have changed", func() { + mockAlertState = &m.AlertState{ + NewState: alertstates.Critical, + } + mockResult.State = alertstates.Ok + So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue) + }) + + Convey("last alert state was 15min ago", func() { + now := time.Now() + mockAlertState = &m.AlertState{ + NewState: alertstates.Critical, + Created: now.Add(time.Minute * -30), + } + mockResult.State = alertstates.Critical + So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue) + }) + }) + }) +}