Merge pull request #11260 from iksaif/master

alertmanager: handle resolved alerts, nodata, and execution errors + cleanups

pkg/services/alerting/notifiers/alertmanager.go

@@ -46,25 +46,49 @@ type AlertmanagerNotifier struct {
 }
 
 func (this *AlertmanagerNotifier) ShouldNotify(evalContext *alerting.EvalContext) bool {
+	this.log.Debug("Should notify", "ruleId", evalContext.Rule.Id, "state", evalContext.Rule.State, "previousState", evalContext.PrevAlertState)
+
+	// Do not notify when we become OK for the first time.
+	if (evalContext.PrevAlertState == m.AlertStatePending) && (evalContext.Rule.State == m.AlertStateOK) {
+		return false
+	}
+	// Notify on Alerting -> OK to resolve before alertmanager timeout.
+	if (evalContext.PrevAlertState == m.AlertStateAlerting) && (evalContext.Rule.State == m.AlertStateOK) {
+		return true
+	}
 	return evalContext.Rule.State == m.AlertStateAlerting
 }
 
-func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error {
+func (this *AlertmanagerNotifier) createAlert(evalContext *alerting.EvalContext, match *alerting.EvalMatch, ruleUrl string) *simplejson.Json {
+	alertJSON := simplejson.New()
+	alertJSON.Set("startsAt", evalContext.StartTime.UTC().Format(time.RFC3339))
+	if evalContext.Rule.State == m.AlertStateOK {
+		alertJSON.Set("endsAt", time.Now().UTC().Format(time.RFC3339))
+	}
+	alertJSON.Set("generatorURL", ruleUrl)
 
-	alerts := make([]interface{}, 0)
-	for _, match := range evalContext.EvalMatches {
-		alertJSON := simplejson.New()
-		alertJSON.Set("startsAt", evalContext.StartTime.UTC().Format(time.RFC3339))
-
-		if ruleUrl, err := evalContext.GetRuleUrl(); err == nil {
-			alertJSON.Set("generatorURL", ruleUrl)
+	// Annotations (summary and description are very commonly used).
+	alertJSON.SetPath([]string{"annotations", "summary"}, evalContext.Rule.Name)
+	description := ""
+	if evalContext.Rule.Message != "" {
+		description += evalContext.Rule.Message
+	}
+	if evalContext.Error != nil {
+		if description != "" {
+			description += "\n"
 		}
+		description += "Error: " + evalContext.Error.Error()
+	}
+	if description != "" {
+		alertJSON.SetPath([]string{"annotations", "description"}, description)
+	}
+	if evalContext.ImagePublicUrl != "" {
+		alertJSON.SetPath([]string{"annotations", "image"}, evalContext.ImagePublicUrl)
+	}
 
-		if evalContext.Rule.Message != "" {
-			alertJSON.SetPath([]string{"annotations", "description"}, evalContext.Rule.Message)
-		}
-
-		tags := make(map[string]string)
+	// Labels (from metrics tags + mandatory alertname).
+	tags := make(map[string]string)
+	if match != nil {
 		if len(match.Tags) == 0 {
 			tags["metric"] = match.Metric
 		} else {
@@ -72,10 +96,32 @@ func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) erro
 				tags[k] = v
 			}
 		}
-		tags["alertname"] = evalContext.Rule.Name
-		alertJSON.Set("labels", tags)
+	}
+	tags["alertname"] = evalContext.Rule.Name
+	alertJSON.Set("labels", tags)
+	return alertJSON
+}
 
-		alerts = append(alerts, alertJSON)
+func (this *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error {
+	this.log.Info("Sending Alertmanager alert", "ruleId", evalContext.Rule.Id, "notification", this.Name)
+
+	ruleUrl, err := evalContext.GetRuleUrl()
+	if err != nil {
+		this.log.Error("Failed get rule link", "error", err)
+		return err
+	}
+
+	// Send one alert per matching series.
+	alerts := make([]interface{}, 0)
+	for _, match := range evalContext.EvalMatches {
+		alert := this.createAlert(evalContext, match, ruleUrl)
+		alerts = append(alerts, alert)
+	}
+
+	// This happens on ExecutionError or NoData
+	if len(alerts) == 0 {
+		alert := this.createAlert(evalContext, nil, ruleUrl)
+		alerts = append(alerts, alert)
 	}
 
 	bodyJSON := simplejson.NewFromAny(alerts)

pkg/services/alerting/notifiers/base.go

@@ -27,15 +27,21 @@ func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model
 }
 
 func defaultShouldNotify(context *alerting.EvalContext) bool {
+	// Only notify on state change.
 	if context.PrevAlertState == context.Rule.State {
 		return false
 	}
+	// Do not notify when we become OK for the first time.
 	if (context.PrevAlertState == m.AlertStatePending) && (context.Rule.State == m.AlertStateOK) {
 		return false
 	}
 	return true
 }
 
+func (n *NotifierBase) ShouldNotify(context *alerting.EvalContext) bool {
+	return defaultShouldNotify(context)
+}
+
 func (n *NotifierBase) GetType() string {
 	return n.Type
 }

pkg/services/alerting/notifiers/dingding.go

@@ -38,10 +38,6 @@ func NewDingDingNotifier(model *m.AlertNotification) (alerting.Notifier, error)
 	}, nil
 }
 
-func (this *DingDingNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 type DingDingNotifier struct {
 	NotifierBase
 	Url string

pkg/services/alerting/notifiers/email.go

@@ -58,10 +58,6 @@ func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}, nil
 }
 
-func (this *EmailNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *EmailNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Sending alert notification to", "addresses", this.Addresses)
 

pkg/services/alerting/notifiers/hipchat.go

@@ -75,10 +75,6 @@ type HipChatNotifier struct {
 	log    log.Logger
 }
 
-func (this *HipChatNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *HipChatNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Executing hipchat notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
 

pkg/services/alerting/notifiers/kafka.go

@@ -57,10 +57,6 @@ type KafkaNotifier struct {
 	log      log.Logger
 }
 
-func (this *KafkaNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *KafkaNotifier) Notify(evalContext *alerting.EvalContext) error {
 
 	state := evalContext.Rule.State

pkg/services/alerting/notifiers/line.go

@@ -51,10 +51,6 @@ type LineNotifier struct {
 	log   log.Logger
 }
 
-func (this *LineNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *LineNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Executing line notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
 

pkg/services/alerting/notifiers/opsgenie.go

@@ -72,10 +72,6 @@ type OpsGenieNotifier struct {
 	log       log.Logger
 }
 
-func (this *OpsGenieNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *OpsGenieNotifier) Notify(evalContext *alerting.EvalContext) error {
 
 	var err error

pkg/services/alerting/notifiers/pagerduty.go

@@ -65,10 +65,6 @@ type PagerdutyNotifier struct {
 	log         log.Logger
 }
 
-func (this *PagerdutyNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error {
 
 	if evalContext.Rule.State == m.AlertStateOK && !this.AutoResolve {

pkg/services/alerting/notifiers/pushover.go

@@ -123,10 +123,6 @@ type PushoverNotifier struct {
 	log      log.Logger
 }
 
-func (this *PushoverNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *PushoverNotifier) Notify(evalContext *alerting.EvalContext) error {
 	ruleUrl, err := evalContext.GetRuleUrl()
 	if err != nil {

pkg/services/alerting/notifiers/sensu.go

@@ -71,10 +71,6 @@ type SensuNotifier struct {
 	log      log.Logger
 }
 
-func (this *SensuNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *SensuNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Sending sensu result")
 

pkg/services/alerting/notifiers/slack.go

@@ -98,10 +98,6 @@ type SlackNotifier struct {
 	log       log.Logger
 }
 
-func (this *SlackNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *SlackNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Executing slack notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
 

pkg/services/alerting/notifiers/teams.go

@@ -47,10 +47,6 @@ type TeamsNotifier struct {
 	log       log.Logger
 }
 
-func (this *TeamsNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *TeamsNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Executing teams notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)
 

pkg/services/alerting/notifiers/telegram.go

@@ -208,6 +208,7 @@ func generateImageCaption(evalContext *alerting.EvalContext, ruleUrl string, met
 
 	return message
 }
+
 func appendIfPossible(message string, extra string, sizeLimit int) string {
 	if len(extra)+len(message) <= sizeLimit {
 		return message + extra
@@ -216,10 +217,6 @@ func appendIfPossible(message string, extra string, sizeLimit int) string {
 	return message
 }
 
-func (this *TelegramNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *TelegramNotifier) Notify(evalContext *alerting.EvalContext) error {
 	var cmd *m.SendWebhookSync
 	if evalContext.ImagePublicUrl == "" && this.UploadImage == true {

pkg/services/alerting/notifiers/threema.go

@@ -114,10 +114,6 @@ func NewThreemaNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}, nil
 }
 
-func (this *ThreemaNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (notifier *ThreemaNotifier) Notify(evalContext *alerting.EvalContext) error {
 	notifier.log.Info("Sending alert notification from", "threema_id", notifier.GatewayID)
 	notifier.log.Info("Sending alert notification to", "threema_id", notifier.RecipientID)

pkg/services/alerting/notifiers/victorops.go

@@ -68,10 +68,6 @@ type VictoropsNotifier struct {
 	log         log.Logger
 }
 
-func (this *VictoropsNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 // Notify sends notification to Victorops via POST to URL endpoint
 func (this *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.Id, "notification", this.Name)

pkg/services/alerting/notifiers/webhook.go

@@ -65,10 +65,6 @@ type WebhookNotifier struct {
 	log        log.Logger
 }
 
-func (this *WebhookNotifier) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
-}
-
 func (this *WebhookNotifier) Notify(evalContext *alerting.EvalContext) error {
 	this.log.Info("Sending webhook")