IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-20 11:48:34 -06:00
Code Issues Packages Projects Releases Wiki Activity

feat(alerting): lots of progress on notifications, refactored them out to their own package, restored webhook notitication and added slack notification

Browse Source
This commit is contained in:
Torkel Ödegaard 2016-07-27 12:09:55 +02:00
parent 77c66a88d9
commit ae5f8a76d9
16 changed files with 416 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/cmd/grafana-server/main.go
View File

@ -16,7 +16,7 @@ import (
"github.com/grafana/grafana/pkg/login"
"github.com/grafana/grafana/pkg/metrics"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/services/alerting"
alertingInit "github.com/grafana/grafana/pkg/services/alerting/init"
"github.com/grafana/grafana/pkg/services/eventpublisher"
"github.com/grafana/grafana/pkg/services/notifications"
"github.com/grafana/grafana/pkg/services/search"
@ -68,7 +68,7 @@ func main() {
social.NewOAuthService()
eventpublisher.Init()
plugins.Init()
alerting.Init()
alertingInit.Init()
if err := notifications.Init(); err != nil {
log.Fatal(3, "Notification service failed to initialize", err)

21
pkg/services/alerting/alerting.go
View File

@ -1,21 +0,0 @@
package alerting
import (
"github.com/grafana/grafana/pkg/setting"
_ "github.com/grafana/grafana/pkg/tsdb/graphite"
)
var (
maxAlertExecutionRetries = 3
)
var engine *Engine
func Init() {
if !setting.AlertingEnabled {
return
}
engine = NewEngine()
engine.Start()
}

19
pkg/services/alerting/init/init.go Normal file
View File

@ -0,0 +1,19 @@
package init
import (
"github.com/grafana/grafana/pkg/services/alerting"
_ "github.com/grafana/grafana/pkg/services/alerting/notifiers"
"github.com/grafana/grafana/pkg/setting"
_ "github.com/grafana/grafana/pkg/tsdb/graphite"
)
var engine *alerting.Engine
func Init() {
if !setting.AlertingEnabled {
return
}
engine = alerting.NewEngine()
engine.Start()
}

21
pkg/services/alerting/models.go
View File

@ -8,23 +8,10 @@ import (
)
type AlertJob struct {
Offset int64
Delay bool
Running bool
RetryCount int
Rule *AlertRule
}
func (aj *AlertJob) Retryable() bool {
return aj.RetryCount < maxAlertExecutionRetries
}
func (aj *AlertJob) ResetRetry() {
aj.RetryCount = 0
}
func (aj *AlertJob) IncRetry() {
aj.RetryCount++
Offset int64
Delay bool
Running bool
Rule *AlertRule
}
type AlertResultContext struct {

112
pkg/services/alerting/notifier.go
View File

@ -2,17 +2,13 @@ package alerting
import (
"errors"
"fmt"
"strings"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
)
type RootNotifier struct {
NotifierBase
log log.Logger
}
@ -22,6 +18,10 @@ func NewRootNotifier() *RootNotifier {
}
}
func (n *RootNotifier) GetType() string {
return "root"
}
func (n *RootNotifier) Notify(context *AlertResultContext) {
n.log.Info("Sending notifications for", "ruleId", context.Rule.Id)
@ -46,7 +46,7 @@ func (n *RootNotifier) getNotifiers(orgId int64, notificationIds []int64) ([]Not
var result []Notifier
for _, notification := range query.Result {
if not, err := NewNotificationFromDBModel(notification); err != nil {
if not, err := n.getNotifierFor(notification); err != nil {
return nil, err
} else {
result = append(result, not)
@ -56,47 +56,40 @@ func (n *RootNotifier) getNotifiers(orgId int64, notificationIds []int64) ([]Not
return result, nil
}
type NotifierBase struct {
Name string
Type string
}
func (n *NotifierBase) GetType() string {
return n.Type
}
type EmailNotifier struct {
NotifierBase
Addresses []string
log log.Logger
}
func (this *EmailNotifier) Notify(context *AlertResultContext) {
this.log.Info("Sending alert notification to", "addresses", this.Addresses)
slugQuery := &m.GetDashboardSlugByIdQuery{Id: context.Rule.DashboardId}
if err := bus.Dispatch(slugQuery); err != nil {
this.log.Error("Failed to load dashboard", "error", err)
return
func (n *RootNotifier) getNotifierFor(model *m.AlertNotification) (Notifier, error) {
factory, found := notifierFactories[model.Type]
if !found {
return nil, errors.New("Unsupported notification type")
}
ruleLink := fmt.Sprintf("%sdashboard/db/%s?fullscreen&edit&tab=alert&panelId=%d", setting.AppUrl, slugQuery.Result, context.Rule.PanelId)
return factory(model)
// if model.Type == "email" {
// addressesString := model.Settings.Get("addresses").MustString()
//
// if addressesString == "" {
// return nil, fmt.Errorf("Could not find addresses in settings")
// }
//
// NotifierBase: NotifierBase{
// Name: model.Name,
// Type: model.Type,
// },
// Addresses: strings.Split(addressesString, "\n"),
// log: log.New("alerting.notification.email"),
// }, nil
// }
cmd := &m.SendEmailCommand{
Data: map[string]interface{}{
"RuleState": context.Rule.State,
"RuleName": context.Rule.Name,
"Severity": context.Rule.Severity,
"RuleLink": ruleLink,
},
To: this.Addresses,
Template: "alert_notification.html",
}
err := bus.Dispatch(cmd)
if err != nil {
this.log.Error("Failed to send alert notification email", "error", err)
}
// url := settings.Get("url").MustString()
// if url == "" {
// return nil, fmt.Errorf("Could not find url propertie in settings")
// }
//
// return &WebhookNotifier{
// Url: url,
// User: settings.Get("user").MustString(),
// Password: settings.Get("password").MustString(),
// log: log.New("alerting.notification.webhook"),
// }, nil
}
// type WebhookNotifier struct {
@ -126,35 +119,10 @@ func (this *EmailNotifier) Notify(context *AlertResultContext) {
// bus.Dispatch(cmd)
// }
func NewNotificationFromDBModel(model *m.AlertNotification) (Notifier, error) {
if model.Type == "email" {
addressesString := model.Settings.Get("addresses").MustString()
type NotifierFactory func(notification *m.AlertNotification) (Notifier, error)
if addressesString == "" {
return nil, fmt.Errorf("Could not find addresses in settings")
}
var notifierFactories map[string]NotifierFactory = make(map[string]NotifierFactory)
return &EmailNotifier{
NotifierBase: NotifierBase{
Name: model.Name,
Type: model.Type,
},
Addresses: strings.Split(addressesString, "\n"),
log: log.New("alerting.notification.email"),
}, nil
}
return nil, errors.New("Unsupported notification type")
// url := settings.Get("url").MustString()
// if url == "" {
// return nil, fmt.Errorf("Could not find url propertie in settings")
// }
//
// return &WebhookNotifier{
// Url: url,
// User: settings.Get("user").MustString(),
// Password: settings.Get("password").MustString(),
// log: log.New("alerting.notification.webhook"),
// }, nil
func RegisterNotifier(typeName string, factory NotifierFactory) {
notifierFactories[typeName] = factory
}

10
pkg/services/alerting/notifiers/base.go Normal file
View File

@ -0,0 +1,10 @@
package notifiers
type NotifierBase struct {
Name string
Type string
}
func (n *NotifierBase) GetType() string {
return n.Type
}

20
pkg/services/alerting/notifiers/common.go Normal file
View File

@ -0,0 +1,20 @@
package notifiers
import (
"fmt"
"github.com/grafana/grafana/pkg/bus"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting"
"github.com/grafana/grafana/pkg/setting"
)
func getRuleLink(rule *alerting.AlertRule) (string, error) {
slugQuery := &m.GetDashboardSlugByIdQuery{Id: rule.DashboardId}
if err := bus.Dispatch(slugQuery); err != nil {
return "", err
}
ruleLink := fmt.Sprintf("%sdashboard/db/%s?fullscreen&edit&tab=alert&panelId=%d", setting.AppUrl, slugQuery.Result, rule.PanelId)
return ruleLink, nil
}

62
pkg/services/alerting/notifiers/email.go Normal file
View File

@ -0,0 +1,62 @@
package notifiers
import (
"strings"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting"
)
func init() {
alerting.RegisterNotifier("email", NewEmailNotifier)
}
type EmailNotifier struct {
NotifierBase
Addresses []string
log log.Logger
}
func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
addressesString := model.Settings.Get("addresses").MustString()
if addressesString == "" {
return nil, alerting.AlertValidationError{Reason: "Could not find addresses in settings"}
}
return &EmailNotifier{
NotifierBase: NotifierBase{
Name: model.Name,
Type: model.Type,
},
Addresses: strings.Split(addressesString, "\n"),
log: log.New("alerting.notifier.email"),
}, nil
}
func (this *EmailNotifier) Notify(context *alerting.AlertResultContext) {
this.log.Info("Sending alert notification to", "addresses", this.Addresses)
ruleLink, err := getRuleLink(context.Rule)
if err != nil {
this.log.Error("Failed get rule link", "error", err)
return
}
cmd := &m.SendEmailCommand{
Data: map[string]interface{}{
"RuleState": context.Rule.State,
"RuleName": context.Rule.Name,
"Severity": context.Rule.Severity,
"RuleLink": ruleLink,
},
To: this.Addresses,
Template: "alert_notification.html",
}
if err := bus.Dispatch(cmd); err != nil {
this.log.Error("Failed to send alert notification email", "error", err)
}
}

52
pkg/services/alerting/notifiers/email_test.go Normal file
View File

@ -0,0 +1,52 @@
package notifiers
import (
"testing"
"github.com/grafana/grafana/pkg/components/simplejson"
m "github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
func TestEmailNotifier(t *testing.T) {
Convey("Email notifier tests", t, func() {
Convey("Parsing alert notification from settings", func() {
Convey("empty settings should return error", func() {
json := `{ }`
settingsJSON, _ := simplejson.NewJson([]byte(json))
model := &m.AlertNotification{
Name: "ops",
Type: "email",
Settings: settingsJSON,
}
_, err := NewEmailNotifier(model)
So(err, ShouldNotBeNil)
})
Convey("from settings", func() {
json := `
{
"addresses": "ops@grafana.org"
}`
settingsJSON, _ := simplejson.NewJson([]byte(json))
model := &m.AlertNotification{
Name: "ops",
Type: "email",
Settings: settingsJSON,
}
not, err := NewEmailNotifier(model)
emailNotifier := not.(*EmailNotifier)
So(err, ShouldBeNil)
So(emailNotifier.Name, ShouldEqual, "ops")
So(emailNotifier.Type, ShouldEqual, "email")
So(emailNotifier.Addresses[0], ShouldEqual, "ops@grafana.org")
})
})
})
}

66
pkg/services/alerting/notifiers/slack.go Normal file
View File

@ -0,0 +1,66 @@
package notifiers
import (
"fmt"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/log"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting"
)
func init() {
alerting.RegisterNotifier("slack", NewSlackNotifier)
}
func NewSlackNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
url := model.Settings.Get("url").MustString()
if url == "" {
return nil, alerting.AlertValidationError{Reason: "Could not find url property in settings"}
}
return &SlackNotifier{
NotifierBase: NotifierBase{
Name: model.Name,
Type: model.Type,
},
Url: url,
log: log.New("alerting.notifier.slack"),
}, nil
}
type SlackNotifier struct {
NotifierBase
Url string
log log.Logger
}
func (this *SlackNotifier) Notify(context *alerting.AlertResultContext) {
this.log.Info("Executing slack notification", "ruleId", context.Rule.Id, "notification", this.Name)
rule := context.Rule
ruleLink, err := getRuleLink(rule)
if err != nil {
this.log.Error("Failed get rule link", "error", err)
return
}
stateText := string(rule.Severity)
if !context.Firing {
stateText = "ok"
}
text := fmt.Sprintf("[%s]: <%s|%s>", stateText, ruleLink, rule.Name)
body := simplejson.New()
body.Set("text", text)
data, _ := body.MarshalJSON()
cmd := &m.SendWebhook{Url: this.Url, Body: string(data)}
if err := bus.Dispatch(cmd); err != nil {
this.log.Error("Failed to send slack notification", "error", err, "webhook", this.Name)
}
}

61
pkg/services/alerting/notifiers/webhook.go Normal file
View File

@ -0,0 +1,61 @@
package notifiers
import (
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/log"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting"
)
func init() {
alerting.RegisterNotifier("webhook", NewWebHookNotifier)
}
func NewWebHookNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
url := model.Settings.Get("url").MustString()
if url == "" {
return nil, alerting.AlertValidationError{Reason: "Could not find url property in settings"}
}
return &WebhookNotifier{
NotifierBase: NotifierBase{
Name: model.Name,
Type: model.Type,
},
Url: url,
User: model.Settings.Get("user").MustString(),
Password: model.Settings.Get("password").MustString(),
log: log.New("alerting.notifier.webhook"),
}, nil
}
type WebhookNotifier struct {
NotifierBase
Url string
User string
Password string
log log.Logger
}
func (this *WebhookNotifier) Notify(context *alerting.AlertResultContext) {
this.log.Info("Sending webhook")
bodyJSON := simplejson.New()
bodyJSON.Set("name", context.Rule.Name)
bodyJSON.Set("firing", context.Firing)
bodyJSON.Set("severity", context.Rule.Severity)
body, _ := bodyJSON.MarshalJSON()
cmd := &m.SendWebhook{
Url: this.Url,
User: this.User,
Password: this.Password,
Body: string(body),
}
if err := bus.Dispatch(cmd); err != nil {
this.log.Error("Failed to send webhook", "error", err, "webhook", this.Name)
}
}

52
pkg/services/alerting/notifiers/webhook_test.go Normal file
View File

@ -0,0 +1,52 @@
package notifiers
import (
"testing"
"github.com/grafana/grafana/pkg/components/simplejson"
m "github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
func TestWebhookNotifier(t *testing.T) {
Convey("Webhook notifier tests", t, func() {
Convey("Parsing alert notification from settings", func() {
Convey("empty settings should return error", func() {
json := `{ }`
settingsJSON, _ := simplejson.NewJson([]byte(json))
model := &m.AlertNotification{
Name: "ops",
Type: "email",
Settings: settingsJSON,
}
_, err := NewWebHookNotifier(model)
So(err, ShouldNotBeNil)
})
Convey("from settings", func() {
json := `
{
"url": "http://google.com"
}`
settingsJSON, _ := simplejson.NewJson([]byte(json))
model := &m.AlertNotification{
Name: "ops",
Type: "email",
Settings: settingsJSON,
}
not, err := NewWebHookNotifier(model)
emailNotifier := not.(*WebhookNotifier)
So(err, ShouldBeNil)
So(emailNotifier.Name, ShouldEqual, "ops")
So(emailNotifier.Type, ShouldEqual, "email")
So(emailNotifier.Url, ShouldEqual, "http://google.com")
})
})
})
}

3
pkg/services/alerting/scheduler.go
View File

@ -29,8 +29,7 @@ func (s *SchedulerImpl) Update(alerts []*AlertRule) {
job = s.jobs[rule.Id]
} else {
job = &AlertJob{
Running: false,
RetryCount: 0,
Running: false,
}
}

15
pkg/services/notifications/webhook.go
View File

@ -2,6 +2,8 @@ package notifications
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"time"
@ -39,6 +41,8 @@ func processWebhookQueue() {
}
func sendWebRequest(webhook *Webhook) error {
webhookLog.Debug("Sending webhook", "url", webhook.Url)
client := http.Client{
Timeout: time.Duration(3 * time.Second),
}
@ -56,8 +60,17 @@ func sendWebRequest(webhook *Webhook) error {
if err != nil {
return err
}
defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode != 200 {
return fmt.Errorf("Webhook response code %s", resp.StatusCode)
}
defer resp.Body.Close()
return nil
}

4
public/app/features/alerting/notification_edit_ctrl.ts
View File

@ -41,6 +41,10 @@ export class AlertNotificationEditCtrl {
});
}
}
typeChanged() {
this.model.settings = {};
}
}
coreModule.controller('AlertNotificationEditCtrl', AlertNotificationEditCtrl);

10
public/app/features/alerting/partials/notification_edit.html
View File

@ -20,7 +20,7 @@
<div class="gf-form-select-wrapper width-15">
<select class="gf-form-input"
ng-model="ctrl.model.type"
ng-options="t for t in ['webhook', 'email']"
ng-options="t for t in ['webhook', 'email', 'slack']"
ng-change="ctrl.typeChanged(notification, $index)">
</select>
</div>
@ -45,6 +45,14 @@
</div>
</div>
<div class="gf-form-group" ng-show="ctrl.model.type === 'slack'">
<h3 class="page-heading">Slack settings</h3>
<div class="gf-form">
<span class="gf-form-label width-6">Url</span>
<input type="text" class="gf-form-input max-width-26" ng-model="ctrl.model.settings.url" placeholder="Slack incoming webhook url"></input>
</div>
</div>
<div class="gf-form-group section" ng-show="ctrl.model.type === 'email'">
<h3 class="page-heading">Email addresses</h3>
<div class="gf-form">