diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 002f2369c9b..f2616b3444c 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -41,6 +41,7 @@ var ( M_Alerting_Result_State_Paused Counter M_Alerting_Result_State_NoData Counter M_Alerting_Result_State_ExecError Counter + M_Alerting_Result_State_Initialized Counter M_Alerting_Active_Alerts Counter M_Alerting_Notification_Sent_Slack Counter M_Alerting_Notification_Sent_Email Counter @@ -102,6 +103,7 @@ func initMetricVars(settings *MetricSettings) { M_Alerting_Result_State_Paused = RegCounter("alerting.result", "state", "paused") M_Alerting_Result_State_NoData = RegCounter("alerting.result", "state", "no_data") M_Alerting_Result_State_ExecError = RegCounter("alerting.result", "state", "exec_error") + M_Alerting_Result_State_Initialized = RegCounter("alerting.result", "state", "initialized") M_Alerting_Active_Alerts = RegCounter("alerting.active_alerts") M_Alerting_Notification_Sent_Slack = RegCounter("alerting.notifications_sent", "type", "slack") diff --git a/pkg/models/alert.go b/pkg/models/alert.go index 7531be90e88..b0d97756955 100644 --- a/pkg/models/alert.go +++ b/pkg/models/alert.go @@ -11,11 +11,12 @@ type AlertSeverityType string type NoDataOption string const ( - AlertStateNoData AlertStateType = "no_data" - AlertStateExecError AlertStateType = "execution_error" - AlertStatePaused AlertStateType = "paused" - AlertStateAlerting AlertStateType = "alerting" - AlertStateOK AlertStateType = "ok" + AlertStateNoData AlertStateType = "no_data" + AlertStateExecError AlertStateType = "execution_error" + AlertStatePaused AlertStateType = "paused" + AlertStateAlerting AlertStateType = "alerting" + AlertStateOK AlertStateType = "ok" + AlertStateInitialized AlertStateType = "initialized" ) const ( @@ -26,7 +27,7 @@ const ( ) func (s AlertStateType) IsValid() bool { - return s == AlertStateOK || s == AlertStateNoData || s == AlertStateExecError || s == AlertStatePaused + return s == AlertStateOK || s == AlertStateNoData || s == AlertStateExecError || s == AlertStatePaused || s == AlertStateInitialized } func (s NoDataOption) IsValid() bool { diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index d786e8d599d..323c8776f4a 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -86,7 +86,12 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { handler.log.Error("Failed to save annotation for new alert state", "error", err) } - handler.notifier.Notify(evalContext) + if (oldState == m.AlertStateInitialized) && (evalContext.Rule.State == m.AlertStateOK) { + handler.log.Info("Notfication not sent", "oldState", oldState, "newState", evalContext.Rule.State) + } else { + handler.notifier.Notify(evalContext) + } + } return nil @@ -98,6 +103,8 @@ func (handler *DefaultResultHandler) shouldUpdateAlertState(evalContext *EvalCon func countStateResult(state m.AlertStateType) { switch state { + case m.AlertStateInitialized: + metrics.M_Alerting_Result_State_Initialized.Inc(1) case m.AlertStateAlerting: metrics.M_Alerting_Result_State_Alerting.Inc(1) case m.AlertStateOK: diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go index 4824b000bcb..4eca93a10d6 100644 --- a/pkg/services/sqlstore/alert.go +++ b/pkg/services/sqlstore/alert.go @@ -173,7 +173,7 @@ func upsertAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *xor } else { alert.Updated = time.Now() alert.Created = time.Now() - alert.State = m.AlertStateNoData + alert.State = m.AlertStateInitialized alert.NewStateDate = time.Now() _, err := sess.Insert(alert)