From b9b65cf2d416068bcd2cb25a76298b7e9560536c Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 15 Jun 2016 09:19:22 +0200 Subject: [PATCH] tech(alerting): add logging about failed notifications --- pkg/models/alert.go | 4 +++ pkg/services/alerting/engine.go | 26 +++++++++++----- pkg/services/alerting/interfaces.go | 4 +++ pkg/services/alerting/notifier.go | 48 +++++++++++++++++------------ 4 files changed, 55 insertions(+), 27 deletions(-) diff --git a/pkg/models/alert.go b/pkg/models/alert.go index 148a85cc7c5..64562534e9e 100644 --- a/pkg/models/alert.go +++ b/pkg/models/alert.go @@ -27,6 +27,10 @@ func (alert *Alert) ValidToSave() bool { return alert.DashboardId != 0 && alert.OrgId != 0 && alert.PanelId != 0 } +func (alert *Alert) ShouldUpdateState(newState string) bool { + return alert.State != newState +} + func (this *Alert) ContainsUpdates(other *Alert) bool { result := false result = result || this.Name != other.Name diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 06d323ef14f..458fabd2f6b 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -20,6 +20,7 @@ type Engine struct { handler AlertingHandler ruleReader RuleReader log log.Logger + notifier Notifier } func NewEngine() *Engine { @@ -31,6 +32,7 @@ func NewEngine() *Engine { handler: NewHandler(), ruleReader: NewRuleReader(), log: log.New("alerting.engine"), + notifier: NewNotifier(), } return e @@ -129,13 +131,23 @@ func (e *Engine) resultHandler() { } func (e *Engine) saveState(result *AlertResult) { - cmd := &m.UpdateAlertStateCommand{ - AlertId: result.AlertJob.Rule.Id, - NewState: result.State, - Info: result.Description, - } + query := &m.GetAlertByIdQuery{Id: result.AlertJob.Rule.Id} + bus.Dispatch(query) - if err := bus.Dispatch(cmd); err != nil { - e.log.Error("Failed to save state", "error", err) + if query.Result.ShouldUpdateState(result.State) { + cmd := &m.UpdateAlertStateCommand{ + AlertId: result.AlertJob.Rule.Id, + NewState: result.State, + Info: result.Description, + } + + if err := bus.Dispatch(cmd); err != nil { + e.log.Error("Failed to save state", "error", err) + } + + e.log.Debug("will notify! about", "new state", result.State) + e.notifier.Notify(result) + } else { + e.log.Debug("state remains the same!") } } diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go index e57ed62a9a6..87ded1a2631 100644 --- a/pkg/services/alerting/interfaces.go +++ b/pkg/services/alerting/interfaces.go @@ -10,3 +10,7 @@ type Scheduler interface { Tick(time time.Time, execQueue chan *AlertJob) Update(rules []*AlertRule) } + +type Notifier interface { + Notify(alertResult *AlertResult) +} diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index a843a19b0eb..6dbd4812190 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -1,29 +1,32 @@ package alerting import ( + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/log" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting/alertstates" ) -type Notifier struct { +type NotifierImpl struct { log log.Logger } -func NewNotifier() *Notifier { - return &Notifier{ +func NewNotifier() *NotifierImpl { + return &NotifierImpl{ log: log.New("alerting.notifier"), } } -func (n *Notifier) Notify(alertResult AlertResult) { - notifiers := getNotifiers(alertResult.AlertJob.Rule.OrgId, alertResult.AlertJob.Rule.NotificationGroups) +func (n *NotifierImpl) Notify(alertResult *AlertResult) { + n.log.Warn("LETS NOTIFY!!!!A") + notifiers := n.getNotifiers(alertResult.AlertJob.Rule.OrgId, []int64{1, 2}) for _, notifier := range notifiers { + warn := alertResult.State == alertstates.Warn && notifier.SendWarning crit := alertResult.State == alertstates.Critical && notifier.SendCritical - + n.log.Warn("looopie", "warn", warn, "crit", crit) if warn || crit { n.log.Info("Sending notification", "state", alertResult.State, "type", notifier.Type) go notifier.Notifierr.Notify(alertResult) @@ -44,41 +47,44 @@ type Notification struct { type EmailNotifier struct { To string From string + log log.Logger } -func (this EmailNotifier) Notify(alertResult AlertResult) { +func (this *EmailNotifier) Notify(alertResult *AlertResult) { //bus.dispath to notification package in grafana + this.log.Info("Sending email") } type WebhookNotifier struct { Url string AuthUser string AuthPassword string + log log.Logger } -func (this WebhookNotifier) Notify(alertResult AlertResult) { +func (this *WebhookNotifier) Notify(alertResult *AlertResult) { //bus.dispath to notification package in grafana + this.log.Info("Sending webhook") } type Notifierr interface { - Notify(alertResult AlertResult) + Notify(alertResult *AlertResult) } -func getNotifiers(orgId int64, notificationGroups []int64) []*Notification { - var notifications []*m.AlertNotification - - for _, notificationId := range notificationGroups { - query := m.GetAlertNotificationQuery{ - OrgID: orgId, - Id: notificationId, - } - - notifications = append(notifications, query.Result...) +func (n *NotifierImpl) getNotifiers(orgId int64, notificationGroups []int64) []*Notification { + query := &m.GetAlertNotificationQuery{ + OrgID: orgId, + Ids: notificationGroups, + } + err := bus.Dispatch(query) + if err != nil { + n.log.Error("Failed to read notifications", "error", err) } var result []*Notification - for _, notification := range notifications { + n.log.Warn("query result", "length", len(query.Result)) + for _, notification := range query.Result { not, err := NewNotificationFromDBModel(notification) if err == nil { result = append(result, not) @@ -103,6 +109,7 @@ var createNotifier = func(notificationType string, settings *simplejson.Json) No return &EmailNotifier{ To: settings.Get("to").MustString(), From: settings.Get("from").MustString(), + log: log.New("alerting.notification.email"), } } @@ -110,5 +117,6 @@ var createNotifier = func(notificationType string, settings *simplejson.Json) No Url: settings.Get("url").MustString(), AuthUser: settings.Get("user").MustString(), AuthPassword: settings.Get("password").MustString(), + log: log.New("alerting.notification.webhook"), } }