diff --git a/pkg/infra/metrics/metrics.go b/pkg/infra/metrics/metrics.go index b20cebeccbd..248b3e480e8 100644 --- a/pkg/infra/metrics/metrics.go +++ b/pkg/infra/metrics/metrics.go @@ -80,6 +80,9 @@ var ( // MAlertingNotificationSent is a metric counter for how many alert notifications been sent MAlertingNotificationSent *prometheus.CounterVec + // MAlertingNotificationSent is a metric counter for how many alert notifications that failed + MAlertingNotificationFailed *prometheus.CounterVec + // MAwsCloudWatchGetMetricStatistics is a metric counter for getting metric statistics from aws MAwsCloudWatchGetMetricStatistics prometheus.Counter @@ -290,7 +293,13 @@ func init() { MAlertingNotificationSent = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "alerting_notification_sent_total", - Help: "counter for how many alert notifications been sent", + Help: "counter for how many alert notifications have been sent", + Namespace: exporterName, + }, []string{"type"}) + + MAlertingNotificationFailed = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "alerting_notification_failed_total", + Help: "counter for how many alert notifications have failed", Namespace: exporterName, }, []string{"type"}) @@ -452,6 +461,7 @@ func initMetricVars() { MApiDashboardInsert, MAlertingResultState, MAlertingNotificationSent, + MAlertingNotificationFailed, MAwsCloudWatchGetMetricStatistics, MAwsCloudWatchListMetrics, MAwsCloudWatchGetMetricData, diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index d05129a1354..630c5ed6dae 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -63,6 +63,7 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no if err != nil { n.log.Error("failed to send notification", "uid", notifier.GetNotifierUID(), "error", err) + metrics.MAlertingNotificationFailed.WithLabelValues(notifier.GetType()).Inc() } if evalContext.IsTestRun {