diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index 143c72106fe..d3cb3b817fa 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -120,10 +120,10 @@ func AlertTest(c *middleware.Context, dto dtos.AlertTestCommand) Response { } res := backendCmd.Result - dtoRes := &dtos.AlertTestResult{ Firing: res.Firing, ConditionEvals: res.ConditionEvals, + State: res.Rule.State, } if res.Error != nil { diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go index 22c0c3142a8..4f3df88be9f 100644 --- a/pkg/api/dtos/alerting.go +++ b/pkg/api/dtos/alerting.go @@ -37,6 +37,7 @@ type AlertTestCommand struct { type AlertTestResult struct { Firing bool `json:"firing"` + State m.AlertStateType `json:"state"` ConditionEvals string `json:"conditionEvals"` TimeMs string `json:"timeMs"` Error string `json:"error,omitempty"` diff --git a/pkg/services/alerting/eval_handler.go b/pkg/services/alerting/eval_handler.go index 867226738b8..4958cab097b 100644 --- a/pkg/services/alerting/eval_handler.go +++ b/pkg/services/alerting/eval_handler.go @@ -7,6 +7,7 @@ import ( "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/metrics" + "github.com/grafana/grafana/pkg/models" ) type DefaultEvalHandler struct { @@ -60,6 +61,40 @@ func (e *DefaultEvalHandler) Eval(context *EvalContext) { context.Firing = firing context.NoDataFound = noDataFound context.EndTime = time.Now() + context.Rule.State = e.getNewState(context) + elapsedTime := context.EndTime.Sub(context.StartTime) / time.Millisecond metrics.M_Alerting_Execution_Time.Update(elapsedTime) } + +// This should be move into evalContext once its been refactored. +func (handler *DefaultEvalHandler) getNewState(evalContext *EvalContext) models.AlertStateType { + if evalContext.Error != nil { + handler.log.Error("Alert Rule Result Error", + "ruleId", evalContext.Rule.Id, + "name", evalContext.Rule.Name, + "error", evalContext.Error, + "changing state to", evalContext.Rule.ExecutionErrorState.ToAlertState()) + + if evalContext.Rule.ExecutionErrorState == models.ExecutionErrorKeepState { + return evalContext.PrevAlertState + } else { + return evalContext.Rule.ExecutionErrorState.ToAlertState() + } + } else if evalContext.Firing { + return models.AlertStateAlerting + } else if evalContext.NoDataFound { + handler.log.Info("Alert Rule returned no data", + "ruleId", evalContext.Rule.Id, + "name", evalContext.Rule.Name, + "changing state to", evalContext.Rule.NoDataState.ToAlertState()) + + if evalContext.Rule.NoDataState == models.NoDataKeepState { + return evalContext.PrevAlertState + } else { + return evalContext.Rule.NoDataState.ToAlertState() + } + } + + return models.AlertStateOK +} diff --git a/pkg/services/alerting/eval_handler_test.go b/pkg/services/alerting/eval_handler_test.go index 73d675e3fa7..82cc5d43baf 100644 --- a/pkg/services/alerting/eval_handler_test.go +++ b/pkg/services/alerting/eval_handler_test.go @@ -2,8 +2,10 @@ package alerting import ( "context" + "fmt" "testing" + "github.com/grafana/grafana/pkg/models" . "github.com/smartystreets/goconvey/convey" ) @@ -18,8 +20,8 @@ func (c *conditionStub) Eval(context *EvalContext) (*ConditionResult, error) { return &ConditionResult{Firing: c.firing, EvalMatches: c.matches, Operator: c.operator, NoDataFound: c.noData}, nil } -func TestAlertingExecutor(t *testing.T) { - Convey("Test alert execution", t, func() { +func TestAlertingEvaluationHandler(t *testing.T) { + Convey("Test alert evaluation handler", t, func() { handler := NewEvalHandler() Convey("Show return triggered with single passing condition", func() { @@ -164,5 +166,73 @@ func TestAlertingExecutor(t *testing.T) { handler.Eval(context) So(context.NoDataFound, ShouldBeTrue) }) + + Convey("EvalHandler can replace alert state based for errors and no_data", func() { + ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}}) + dummieError := fmt.Errorf("dummie error") + Convey("Should update alert state", func() { + + Convey("ok -> alerting", func() { + ctx.PrevAlertState = models.AlertStateOK + ctx.Firing = true + + So(handler.getNewState(ctx), ShouldEqual, models.AlertStateAlerting) + }) + + Convey("ok -> error(alerting)", func() { + ctx.PrevAlertState = models.AlertStateOK + ctx.Error = dummieError + ctx.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting + + ctx.Rule.State = handler.getNewState(ctx) + So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting) + }) + + Convey("ok -> error(keep_last)", func() { + ctx.PrevAlertState = models.AlertStateOK + ctx.Error = dummieError + ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState + + ctx.Rule.State = handler.getNewState(ctx) + So(ctx.Rule.State, ShouldEqual, models.AlertStateOK) + }) + + Convey("pending -> error(keep_last)", func() { + ctx.PrevAlertState = models.AlertStatePending + ctx.Error = dummieError + ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState + + ctx.Rule.State = handler.getNewState(ctx) + So(ctx.Rule.State, ShouldEqual, models.AlertStatePending) + }) + + Convey("ok -> no_data(alerting)", func() { + ctx.PrevAlertState = models.AlertStateOK + ctx.Rule.NoDataState = models.NoDataSetAlerting + ctx.NoDataFound = true + + ctx.Rule.State = handler.getNewState(ctx) + So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting) + }) + + Convey("ok -> no_data(keep_last)", func() { + ctx.PrevAlertState = models.AlertStateOK + ctx.Rule.NoDataState = models.NoDataKeepState + ctx.NoDataFound = true + + ctx.Rule.State = handler.getNewState(ctx) + So(ctx.Rule.State, ShouldEqual, models.AlertStateOK) + }) + + Convey("pending -> no_data(keep_last)", func() { + ctx.PrevAlertState = models.AlertStatePending + ctx.Rule.NoDataState = models.NoDataKeepState + ctx.NoDataFound = true + + ctx.Rule.State = handler.getNewState(ctx) + So(ctx.Rule.State, ShouldEqual, models.AlertStatePending) + }) + }) + }) }) } diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index aab109088cb..8a78ca2a749 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -27,43 +27,10 @@ func NewResultHandler() *DefaultResultHandler { } } -func (handler *DefaultResultHandler) GetStateFromEvaluation(evalContext *EvalContext) m.AlertStateType { - if evalContext.Error != nil { - handler.log.Error("Alert Rule Result Error", - "ruleId", evalContext.Rule.Id, - "name", evalContext.Rule.Name, - "error", evalContext.Error, - "changing state to", evalContext.Rule.ExecutionErrorState.ToAlertState()) - - if evalContext.Rule.ExecutionErrorState == m.ExecutionErrorKeepState { - return evalContext.PrevAlertState - } else { - return evalContext.Rule.ExecutionErrorState.ToAlertState() - } - } else if evalContext.Firing { - return m.AlertStateAlerting - } else if evalContext.NoDataFound { - handler.log.Info("Alert Rule returned no data", - "ruleId", evalContext.Rule.Id, - "name", evalContext.Rule.Name, - "changing state to", evalContext.Rule.NoDataState.ToAlertState()) - - if evalContext.Rule.NoDataState == m.NoDataKeepState { - return evalContext.PrevAlertState - } else { - return evalContext.Rule.NoDataState.ToAlertState() - } - } - - return m.AlertStateOK -} - func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { executionError := "" annotationData := simplejson.New() - evalContext.Rule.State = handler.GetStateFromEvaluation(evalContext) - if evalContext.Error != nil { executionError = evalContext.Error.Error() annotationData.Set("errorMessage", executionError) diff --git a/pkg/services/alerting/result_handler_test.go b/pkg/services/alerting/result_handler_test.go deleted file mode 100644 index 321838d4f4c..00000000000 --- a/pkg/services/alerting/result_handler_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package alerting - -import ( - "context" - "testing" - - "fmt" - - "github.com/grafana/grafana/pkg/models" - . "github.com/smartystreets/goconvey/convey" -) - -func TestAlertingResultHandler(t *testing.T) { - Convey("Result handler", t, func() { - ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}}) - dummieError := fmt.Errorf("dummie") - handler := NewResultHandler() - - Convey("Should update alert state", func() { - - Convey("ok -> alerting", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Firing = true - - So(handler.GetStateFromEvaluation(ctx), ShouldEqual, models.AlertStateAlerting) - So(ctx.ShouldUpdateAlertState(), ShouldBeTrue) - }) - - Convey("ok -> error(alerting)", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Error = dummieError - ctx.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting - - ctx.Rule.State = handler.GetStateFromEvaluation(ctx) - So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting) - So(ctx.ShouldUpdateAlertState(), ShouldBeTrue) - }) - - Convey("ok -> error(keep_last)", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Error = dummieError - ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState - - ctx.Rule.State = handler.GetStateFromEvaluation(ctx) - So(ctx.Rule.State, ShouldEqual, models.AlertStateOK) - So(ctx.ShouldUpdateAlertState(), ShouldBeFalse) - }) - - Convey("pending -> error(keep_last)", func() { - ctx.PrevAlertState = models.AlertStatePending - ctx.Error = dummieError - ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState - - ctx.Rule.State = handler.GetStateFromEvaluation(ctx) - So(ctx.Rule.State, ShouldEqual, models.AlertStatePending) - So(ctx.ShouldUpdateAlertState(), ShouldBeFalse) - }) - - Convey("ok -> no_data(alerting)", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Rule.NoDataState = models.NoDataSetAlerting - ctx.NoDataFound = true - - ctx.Rule.State = handler.GetStateFromEvaluation(ctx) - So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting) - So(ctx.ShouldUpdateAlertState(), ShouldBeTrue) - }) - - Convey("ok -> no_data(keep_last)", func() { - ctx.PrevAlertState = models.AlertStateOK - ctx.Rule.NoDataState = models.NoDataKeepState - ctx.NoDataFound = true - - ctx.Rule.State = handler.GetStateFromEvaluation(ctx) - So(ctx.Rule.State, ShouldEqual, models.AlertStateOK) - So(ctx.ShouldUpdateAlertState(), ShouldBeFalse) - }) - - Convey("pending -> no_data(keep_last)", func() { - ctx.PrevAlertState = models.AlertStatePending - ctx.Rule.NoDataState = models.NoDataKeepState - ctx.NoDataFound = true - - ctx.Rule.State = handler.GetStateFromEvaluation(ctx) - So(ctx.Rule.State, ShouldEqual, models.AlertStatePending) - So(ctx.ShouldUpdateAlertState(), ShouldBeFalse) - }) - }) - }) -}