tech(alerting): add logging about failed notifications

pkg/models/alert.go

@@ -27,6 +27,10 @@ func (alert *Alert) ValidToSave() bool {
 	return alert.DashboardId != 0 && alert.OrgId != 0 && alert.PanelId != 0
 }
 
+func (alert *Alert) ShouldUpdateState(newState string) bool {
+	return alert.State != newState
+}
+
 func (this *Alert) ContainsUpdates(other *Alert) bool {
 	result := false
 	result = result || this.Name != other.Name

pkg/services/alerting/engine.go

@@ -20,6 +20,7 @@ type Engine struct {
 	handler     AlertingHandler
 	ruleReader  RuleReader
 	log         log.Logger
+	notifier    Notifier
 }
 
 func NewEngine() *Engine {
@@ -31,6 +32,7 @@ func NewEngine() *Engine {
 		handler:     NewHandler(),
 		ruleReader:  NewRuleReader(),
 		log:         log.New("alerting.engine"),
+		notifier:    NewNotifier(),
 	}
 
 	return e
@@ -129,13 +131,23 @@ func (e *Engine) resultHandler() {
 }
 
 func (e *Engine) saveState(result *AlertResult) {
-	cmd := &m.UpdateAlertStateCommand{
-		AlertId:  result.AlertJob.Rule.Id,
-		NewState: result.State,
-		Info:     result.Description,
-	}
+	query := &m.GetAlertByIdQuery{Id: result.AlertJob.Rule.Id}
+	bus.Dispatch(query)
 
-	if err := bus.Dispatch(cmd); err != nil {
-		e.log.Error("Failed to save state", "error", err)
+	if query.Result.ShouldUpdateState(result.State) {
+		cmd := &m.UpdateAlertStateCommand{
+			AlertId:  result.AlertJob.Rule.Id,
+			NewState: result.State,
+			Info:     result.Description,
+		}
+
+		if err := bus.Dispatch(cmd); err != nil {
+			e.log.Error("Failed to save state", "error", err)
+		}
+
+		e.log.Debug("will notify! about", "new state", result.State)
+		e.notifier.Notify(result)
+	} else {
+		e.log.Debug("state remains the same!")
 	}
 }

pkg/services/alerting/interfaces.go

@@ -10,3 +10,7 @@ type Scheduler interface {
 	Tick(time time.Time, execQueue chan *AlertJob)
 	Update(rules []*AlertRule)
 }
+
+type Notifier interface {
+	Notify(alertResult *AlertResult)
+}

pkg/services/alerting/notifier.go

@@ -1,29 +1,32 @@
 package alerting
 
 import (
+	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/log"
 	m "github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/alerting/alertstates"
 )
 
-type Notifier struct {
+type NotifierImpl struct {
 	log log.Logger
 }
 
-func NewNotifier() *Notifier {
-	return &Notifier{
+func NewNotifier() *NotifierImpl {
+	return &NotifierImpl{
 		log: log.New("alerting.notifier"),
 	}
 }
 
-func (n *Notifier) Notify(alertResult AlertResult) {
-	notifiers := getNotifiers(alertResult.AlertJob.Rule.OrgId, alertResult.AlertJob.Rule.NotificationGroups)
+func (n *NotifierImpl) Notify(alertResult *AlertResult) {
+	n.log.Warn("LETS NOTIFY!!!!A")
+	notifiers := n.getNotifiers(alertResult.AlertJob.Rule.OrgId, []int64{1, 2})
 
 	for _, notifier := range notifiers {
+
 		warn := alertResult.State == alertstates.Warn && notifier.SendWarning
 		crit := alertResult.State == alertstates.Critical && notifier.SendCritical
-
+		n.log.Warn("looopie", "warn", warn, "crit", crit)
 		if warn || crit {
 			n.log.Info("Sending notification", "state", alertResult.State, "type", notifier.Type)
 			go notifier.Notifierr.Notify(alertResult)
@@ -44,41 +47,44 @@ type Notification struct {
 type EmailNotifier struct {
 	To   string
 	From string
+	log  log.Logger
 }
 
-func (this EmailNotifier) Notify(alertResult AlertResult) {
+func (this *EmailNotifier) Notify(alertResult *AlertResult) {
 	//bus.dispath to notification package in grafana
+	this.log.Info("Sending email")
 }
 
 type WebhookNotifier struct {
 	Url          string
 	AuthUser     string
 	AuthPassword string
+	log          log.Logger
 }
 
-func (this WebhookNotifier) Notify(alertResult AlertResult) {
+func (this *WebhookNotifier) Notify(alertResult *AlertResult) {
 	//bus.dispath to notification package in grafana
+	this.log.Info("Sending webhook")
 }
 
 type Notifierr interface {
-	Notify(alertResult AlertResult)
+	Notify(alertResult *AlertResult)
 }
 
-func getNotifiers(orgId int64, notificationGroups []int64) []*Notification {
-	var notifications []*m.AlertNotification
-
-	for _, notificationId := range notificationGroups {
-		query := m.GetAlertNotificationQuery{
-			OrgID: orgId,
-			Id:    notificationId,
-		}
-
-		notifications = append(notifications, query.Result...)
+func (n *NotifierImpl) getNotifiers(orgId int64, notificationGroups []int64) []*Notification {
+	query := &m.GetAlertNotificationQuery{
+		OrgID: orgId,
+		Ids:   notificationGroups,
+	}
+	err := bus.Dispatch(query)
+	if err != nil {
+		n.log.Error("Failed to read notifications", "error", err)
 	}
 
 	var result []*Notification
 
-	for _, notification := range notifications {
+	n.log.Warn("query result", "length", len(query.Result))
+	for _, notification := range query.Result {
 		not, err := NewNotificationFromDBModel(notification)
 		if err == nil {
 			result = append(result, not)
@@ -103,6 +109,7 @@ var createNotifier = func(notificationType string, settings *simplejson.Json) No
 		return &EmailNotifier{
 			To:   settings.Get("to").MustString(),
 			From: settings.Get("from").MustString(),
+			log:  log.New("alerting.notification.email"),
 		}
 	}
 
@@ -110,5 +117,6 @@ var createNotifier = func(notificationType string, settings *simplejson.Json) No
 		Url:          settings.Get("url").MustString(),
 		AuthUser:     settings.Get("user").MustString(),
 		AuthPassword: settings.Get("password").MustString(),
+		log:          log.New("alerting.notification.webhook"),
 	}
 }