From b0620a9d4b53d740aadc1e8e91324b0786f704c2 Mon Sep 17 00:00:00 2001 From: ichekrygin Date: Thu, 27 Oct 2016 12:45:55 -0700 Subject: [PATCH 1/8] Add VictorOps alert notification capability --- pkg/metrics/metrics.go | 3 +- pkg/services/alerting/notifiers/victorops.go | 94 +++++++++++++++++++ .../app/features/alerting/alert_tab_ctrl.ts | 1 + .../alerting/partials/notification_edit.html | 10 +- 4 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 pkg/services/alerting/notifiers/victorops.go diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 25fa43d8d2f..b83c63b0845 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -46,7 +46,7 @@ var ( M_Alerting_Notification_Sent_Email Counter M_Alerting_Notification_Sent_Webhook Counter M_Alerting_Notification_Sent_PagerDuty Counter - + M_Alerting_Notification_Sent_Victorops Counter // Timers M_DataSource_ProxyReq_Timer Timer @@ -110,6 +110,7 @@ func initMetricVars(settings *MetricSettings) { M_Alerting_Notification_Sent_Email = RegCounter("alerting.notifications_sent", "type", "email") M_Alerting_Notification_Sent_Webhook = RegCounter("alerting.notifications_sent", "type", "webhook") M_Alerting_Notification_Sent_PagerDuty = RegCounter("alerting.notifications_sent", "type", "pagerduty") + M_Alerting_Notification_Sent_Victorops = RegCounter("alerting.notifications_sent", "type", "victorops") // Timers M_DataSource_ProxyReq_Timer = RegTimer("api.dataproxy.request.all") diff --git a/pkg/services/alerting/notifiers/victorops.go b/pkg/services/alerting/notifiers/victorops.go new file mode 100644 index 00000000000..4d22de3ab9a --- /dev/null +++ b/pkg/services/alerting/notifiers/victorops.go @@ -0,0 +1,94 @@ +package notifiers + +import ( + "encoding/json" + "time" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/metrics" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting" + "github.com/grafana/grafana/pkg/setting" +) + +// AlertStateCritical - Victorops uses "CRITICAL" string to indicate "Alerting" state +const AlertStateCritical = "CRITICAL" + +func init() { + alerting.RegisterNotifier("victorops", NewVictoropsNotifier) +} + +// NewVictoropsNotifier creates an instance of VictoropsNotifier that +// handles posting notifications to Victorops REST API +func NewVictoropsNotifier(model *models.AlertNotification) (alerting.Notifier, error) { + url := model.Settings.Get("url").MustString() + if url == "" { + return nil, alerting.ValidationError{Reason: "Could not find victorops url property in settings"} + } + + return &VictoropsNotifier{ + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + URL: url, + log: log.New("alerting.notifier.victorops"), + }, nil +} + +// VictoropsNotifier defines URL property for Victorops REST API +// and handles notification process by formatting POST body according to +// Victorops specifications (http://victorops.force.com/knowledgebase/articles/Integration/Alert-Ingestion-API-Documentation/) +type VictoropsNotifier struct { + NotifierBase + URL string + log log.Logger +} + +// Notify sends notification to Victorops via POST to URL endpoint +func (this *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error { + this.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.Id, "notification", this.Name) + metrics.M_Alerting_Notification_Sent_Victorops.Inc(1) + + fields := make([]map[string]interface{}, 0) + fieldLimitCount := 4 + for index, evt := range evalContext.EvalMatches { + fields = append(fields, map[string]interface{}{ + "title": evt.Metric, + "value": evt.Value, + "short": true, + }) + if index > fieldLimitCount { + break + } + } + + if evalContext.Error != nil { + fields = append(fields, map[string]interface{}{ + "title": "Error message", + "value": evalContext.Error.Error(), + "short": false, + }) + } + + messageType := evalContext.Rule.State + if evalContext.Rule.State == models.AlertStateAlerting { // translate 'Alerting' to 'CRITICAL' (Victorops analog) + messageType = AlertStateCritical + } + + body := map[string]interface{}{ + "message_type": messageType, + "entity_id": evalContext.Rule.Name, + "timestamp": time.Now().Unix(), + "state_start_time": evalContext.StartTime.Unix(), + "state_message": evalContext.Rule.Message, + "monitoring_tool": "Grafana v" + setting.BuildVersion, + } + + data, _ := json.Marshal(&body) + cmd := &models.SendWebhook{Url: this.URL, Body: string(data)} + + if err := bus.Dispatch(cmd); err != nil { + this.log.Error("Failed to send victorops notification", "error", err, "webhook", this.Name) + } + + return nil +} diff --git a/public/app/features/alerting/alert_tab_ctrl.ts b/public/app/features/alerting/alert_tab_ctrl.ts index c882a6ac3bf..6573ff27ece 100644 --- a/public/app/features/alerting/alert_tab_ctrl.ts +++ b/public/app/features/alerting/alert_tab_ctrl.ts @@ -89,6 +89,7 @@ export class AlertTabCtrl { switch (type) { case "email": return "fa fa-envelope"; case "slack": return "fa fa-slack"; + case "victorops": return "fa fa-exclamation-triangle"; case "webhook": return "fa fa-cubes"; case "pagerduty": return "fa fa-bullhorn"; } diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index 42f1f553d71..f2563279695 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -19,7 +19,7 @@
Type
-
@@ -87,6 +87,14 @@ +
+

Victorops settings

+
+ Url + +
+
+

Email addresses

From 1e558e3936908ac34050672bd5b3f45046939099 Mon Sep 17 00:00:00 2001 From: ichekrygin Date: Thu, 27 Oct 2016 18:39:13 -0700 Subject: [PATCH 2/8] Fix user-facing text --- public/app/features/alerting/partials/notification_edit.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index f2563279695..ac4afbf56fc 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -88,7 +88,7 @@
-

Victorops settings

+

VictorOps settings

Url From 4ca304b7568782e56bd8cda24dd79851d470cb75 Mon Sep 17 00:00:00 2001 From: ichekrygin Date: Fri, 28 Oct 2016 20:47:58 -0700 Subject: [PATCH 3/8] Change VictorOps fa from `fa-exclamation-triangle` -> `fa-pagelines` --- public/app/features/alerting/alert_tab_ctrl.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/features/alerting/alert_tab_ctrl.ts b/public/app/features/alerting/alert_tab_ctrl.ts index 6573ff27ece..03d3ba5b884 100644 --- a/public/app/features/alerting/alert_tab_ctrl.ts +++ b/public/app/features/alerting/alert_tab_ctrl.ts @@ -89,7 +89,7 @@ export class AlertTabCtrl { switch (type) { case "email": return "fa fa-envelope"; case "slack": return "fa fa-slack"; - case "victorops": return "fa fa-exclamation-triangle"; + case "victorops": return "fa fa-pagelines"; case "webhook": return "fa fa-cubes"; case "pagerduty": return "fa fa-bullhorn"; } From b46cbc5fbe13bac36746520ec54a176bb5c1aa12 Mon Sep 17 00:00:00 2001 From: ichekrygin Date: Sun, 30 Oct 2016 11:14:54 -0700 Subject: [PATCH 4/8] Add option to alert on ExecutionError --- pkg/services/alerting/notifiers/victorops.go | 22 ++++++++++++++----- .../alerting/partials/notification_edit.html | 9 ++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/pkg/services/alerting/notifiers/victorops.go b/pkg/services/alerting/notifiers/victorops.go index 4d22de3ab9a..7809eceadac 100644 --- a/pkg/services/alerting/notifiers/victorops.go +++ b/pkg/services/alerting/notifiers/victorops.go @@ -27,10 +27,13 @@ func NewVictoropsNotifier(model *models.AlertNotification) (alerting.Notifier, e return nil, alerting.ValidationError{Reason: "Could not find victorops url property in settings"} } + alertOnExecError := model.Settings.Get("alertOnExecError").MustBool() + return &VictoropsNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), - URL: url, - log: log.New("alerting.notifier.victorops"), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + URL: url, + AlertOnExecError: alertOnExecError, + log: log.New("alerting.notifier.victorops"), }, nil } @@ -39,13 +42,14 @@ func NewVictoropsNotifier(model *models.AlertNotification) (alerting.Notifier, e // Victorops specifications (http://victorops.force.com/knowledgebase/articles/Integration/Alert-Ingestion-API-Documentation/) type VictoropsNotifier struct { NotifierBase - URL string - log log.Logger + URL string + AlertOnExecError bool + log log.Logger } // Notify sends notification to Victorops via POST to URL endpoint func (this *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error { - this.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.Id, "notification", this.Name) + this.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.Id, "notification", this.Name, "url", this.URL, "foo", this.AlertOnExecError) metrics.M_Alerting_Notification_Sent_Victorops.Inc(1) fields := make([]map[string]interface{}, 0) @@ -74,6 +78,12 @@ func (this *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error { messageType = AlertStateCritical } + if evalContext.Rule.State == models.AlertStateExecError && !this.AlertOnExecError { + return nil + } else { + return nil + } + body := map[string]interface{}{ "message_type": messageType, "entity_id": evalContext.Rule.Name, diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index ac4afbf56fc..23840c1310c 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -93,6 +93,15 @@ Url
+
+ + +
From bbca9861fb4273524297eabe174553fd2c122d88 Mon Sep 17 00:00:00 2001 From: ichekrygin Date: Sun, 30 Oct 2016 11:18:31 -0700 Subject: [PATCH 5/8] Fix return on `nil` and cleanup debug messages --- pkg/services/alerting/notifiers/victorops.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/services/alerting/notifiers/victorops.go b/pkg/services/alerting/notifiers/victorops.go index 7809eceadac..bf0042a53e5 100644 --- a/pkg/services/alerting/notifiers/victorops.go +++ b/pkg/services/alerting/notifiers/victorops.go @@ -49,7 +49,7 @@ type VictoropsNotifier struct { // Notify sends notification to Victorops via POST to URL endpoint func (this *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error { - this.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.Id, "notification", this.Name, "url", this.URL, "foo", this.AlertOnExecError) + this.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.Id, "notification") metrics.M_Alerting_Notification_Sent_Victorops.Inc(1) fields := make([]map[string]interface{}, 0) @@ -80,8 +80,6 @@ func (this *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error { if evalContext.Rule.State == models.AlertStateExecError && !this.AlertOnExecError { return nil - } else { - return nil } body := map[string]interface{}{ From 8f0d51171d78faea62e1c95d7dcc723dfc076d16 Mon Sep 17 00:00:00 2001 From: ichekrygin Date: Wed, 16 Nov 2016 16:13:33 -0800 Subject: [PATCH 6/8] Resolve rebase w/ upstream conflicts. Remove: `Alert on Exec Error` (no longer needed) Add: `ruleUrl` to VictorOps message body. --- pkg/services/alerting/notifiers/victorops.go | 18 ++++++++++-------- .../alerting/partials/notification_edit.html | 9 --------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/pkg/services/alerting/notifiers/victorops.go b/pkg/services/alerting/notifiers/victorops.go index bf0042a53e5..d4dfa8c6678 100644 --- a/pkg/services/alerting/notifiers/victorops.go +++ b/pkg/services/alerting/notifiers/victorops.go @@ -49,9 +49,15 @@ type VictoropsNotifier struct { // Notify sends notification to Victorops via POST to URL endpoint func (this *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error { - this.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.Id, "notification") + this.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.Id, "notification", this.Name) metrics.M_Alerting_Notification_Sent_Victorops.Inc(1) + ruleUrl, err := evalContext.GetRuleUrl() + if err != nil { + this.log.Error("Failed get rule link", "error", err) + return err + } + fields := make([]map[string]interface{}, 0) fieldLimitCount := 4 for index, evt := range evalContext.EvalMatches { @@ -78,23 +84,19 @@ func (this *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error { messageType = AlertStateCritical } - if evalContext.Rule.State == models.AlertStateExecError && !this.AlertOnExecError { - return nil - } - body := map[string]interface{}{ "message_type": messageType, "entity_id": evalContext.Rule.Name, "timestamp": time.Now().Unix(), "state_start_time": evalContext.StartTime.Unix(), - "state_message": evalContext.Rule.Message, + "state_message": evalContext.Rule.Message + "\n" + ruleUrl, "monitoring_tool": "Grafana v" + setting.BuildVersion, } data, _ := json.Marshal(&body) - cmd := &models.SendWebhook{Url: this.URL, Body: string(data)} + cmd := &models.SendWebhookSync{Url: this.URL, Body: string(data)} - if err := bus.Dispatch(cmd); err != nil { + if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { this.log.Error("Failed to send victorops notification", "error", err, "webhook", this.Name) } diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index 23840c1310c..ac4afbf56fc 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -93,15 +93,6 @@ Url
-
- - -
From acc729131abe449d9d19e954961343e5470814d5 Mon Sep 17 00:00:00 2001 From: ichekrygin Date: Wed, 16 Nov 2016 16:14:02 -0800 Subject: [PATCH 7/8] Add VictorOps Test. --- .../alerting/notifiers/victorops_test.go | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 pkg/services/alerting/notifiers/victorops_test.go diff --git a/pkg/services/alerting/notifiers/victorops_test.go b/pkg/services/alerting/notifiers/victorops_test.go new file mode 100644 index 00000000000..6ac806a82cc --- /dev/null +++ b/pkg/services/alerting/notifiers/victorops_test.go @@ -0,0 +1,52 @@ +package notifiers + +import ( + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + m "github.com/grafana/grafana/pkg/models" + . "github.com/smartystreets/goconvey/convey" +) + +func TestVictoropsNotifier(t *testing.T) { + Convey("Victorops notifier tests", t, func() { + + Convey("Parsing alert notification from settings", func() { + Convey("empty settings should return error", func() { + json := `{ }` + + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "victorops_testing", + Type: "victorops", + Settings: settingsJSON, + } + + _, err := NewVictoropsNotifier(model) + So(err, ShouldNotBeNil) + }) + + Convey("from settings", func() { + json := ` + { + "url": "http://google.com" + }` + + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &m.AlertNotification{ + Name: "victorops_testing", + Type: "victorops", + Settings: settingsJSON, + } + + not, err := NewVictoropsNotifier(model) + victoropsNotifier := not.(*VictoropsNotifier) + + So(err, ShouldBeNil) + So(victoropsNotifier.Name, ShouldEqual, "victorops_testing") + So(victoropsNotifier.Type, ShouldEqual, "victorops") + So(victoropsNotifier.URL, ShouldEqual, "http://google.com") + }) + }) + }) +} From f6b70c7d8265c30b0821555c91c6ee0169a37797 Mon Sep 17 00:00:00 2001 From: ichekrygin Date: Wed, 16 Nov 2016 17:30:42 -0800 Subject: [PATCH 8/8] Fix formatting --- pkg/metrics/metrics.go | 74 +++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index b83c63b0845..6780908adf7 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -9,43 +9,43 @@ func init() { } var ( - M_Instance_Start Counter - M_Page_Status_200 Counter - M_Page_Status_500 Counter - M_Page_Status_404 Counter - M_Page_Status_Unknown Counter - M_Api_Status_200 Counter - M_Api_Status_404 Counter - M_Api_Status_500 Counter - M_Api_Status_Unknown Counter - M_Proxy_Status_200 Counter - M_Proxy_Status_404 Counter - M_Proxy_Status_500 Counter - M_Proxy_Status_Unknown Counter - M_Api_User_SignUpStarted Counter - M_Api_User_SignUpCompleted Counter - M_Api_User_SignUpInvite Counter - M_Api_Dashboard_Save Timer - M_Api_Dashboard_Get Timer - M_Api_Dashboard_Search Timer - M_Api_Admin_User_Create Counter - M_Api_Login_Post Counter - M_Api_Login_OAuth Counter - M_Api_Org_Create Counter - M_Api_Dashboard_Snapshot_Create Counter - M_Api_Dashboard_Snapshot_External Counter - M_Api_Dashboard_Snapshot_Get Counter - M_Models_Dashboard_Insert Counter - M_Alerting_Result_State_Alerting Counter - M_Alerting_Result_State_Ok Counter - M_Alerting_Result_State_Paused Counter - M_Alerting_Result_State_NoData Counter - M_Alerting_Result_State_Pending Counter - M_Alerting_Active_Alerts Counter - M_Alerting_Notification_Sent_Slack Counter - M_Alerting_Notification_Sent_Email Counter - M_Alerting_Notification_Sent_Webhook Counter - M_Alerting_Notification_Sent_PagerDuty Counter + M_Instance_Start Counter + M_Page_Status_200 Counter + M_Page_Status_500 Counter + M_Page_Status_404 Counter + M_Page_Status_Unknown Counter + M_Api_Status_200 Counter + M_Api_Status_404 Counter + M_Api_Status_500 Counter + M_Api_Status_Unknown Counter + M_Proxy_Status_200 Counter + M_Proxy_Status_404 Counter + M_Proxy_Status_500 Counter + M_Proxy_Status_Unknown Counter + M_Api_User_SignUpStarted Counter + M_Api_User_SignUpCompleted Counter + M_Api_User_SignUpInvite Counter + M_Api_Dashboard_Save Timer + M_Api_Dashboard_Get Timer + M_Api_Dashboard_Search Timer + M_Api_Admin_User_Create Counter + M_Api_Login_Post Counter + M_Api_Login_OAuth Counter + M_Api_Org_Create Counter + M_Api_Dashboard_Snapshot_Create Counter + M_Api_Dashboard_Snapshot_External Counter + M_Api_Dashboard_Snapshot_Get Counter + M_Models_Dashboard_Insert Counter + M_Alerting_Result_State_Alerting Counter + M_Alerting_Result_State_Ok Counter + M_Alerting_Result_State_Paused Counter + M_Alerting_Result_State_NoData Counter + M_Alerting_Result_State_Pending Counter + M_Alerting_Active_Alerts Counter + M_Alerting_Notification_Sent_Slack Counter + M_Alerting_Notification_Sent_Email Counter + M_Alerting_Notification_Sent_Webhook Counter + M_Alerting_Notification_Sent_PagerDuty Counter M_Alerting_Notification_Sent_Victorops Counter // Timers