Instrumentation: Add failed notifications metric (#18089)

This commit is contained in:
Dima Kurguzov 2019-07-24 11:43:24 +02:00 committed by Torkel Ödegaard
parent 9ccf32bef0
commit a4b0ccc138
2 changed files with 12 additions and 1 deletions

View File

@ -80,6 +80,9 @@ var (
// MAlertingNotificationSent is a metric counter for how many alert notifications been sent
MAlertingNotificationSent *prometheus.CounterVec
// MAlertingNotificationSent is a metric counter for how many alert notifications that failed
MAlertingNotificationFailed *prometheus.CounterVec
// MAwsCloudWatchGetMetricStatistics is a metric counter for getting metric statistics from aws
MAwsCloudWatchGetMetricStatistics prometheus.Counter
@ -290,7 +293,13 @@ func init() {
MAlertingNotificationSent = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "alerting_notification_sent_total",
Help: "counter for how many alert notifications been sent",
Help: "counter for how many alert notifications have been sent",
Namespace: exporterName,
}, []string{"type"})
MAlertingNotificationFailed = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "alerting_notification_failed_total",
Help: "counter for how many alert notifications have failed",
Namespace: exporterName,
}, []string{"type"})
@ -452,6 +461,7 @@ func initMetricVars() {
MApiDashboardInsert,
MAlertingResultState,
MAlertingNotificationSent,
MAlertingNotificationFailed,
MAwsCloudWatchGetMetricStatistics,
MAwsCloudWatchListMetrics,
MAwsCloudWatchGetMetricData,

View File

@ -63,6 +63,7 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no
if err != nil {
n.log.Error("failed to send notification", "uid", notifier.GetNotifierUID(), "error", err)
metrics.MAlertingNotificationFailed.WithLabelValues(notifier.GetType()).Inc()
}
if evalContext.IsTestRun {