From b0620a9d4b53d740aadc1e8e91324b0786f704c2 Mon Sep 17 00:00:00 2001 From: ichekrygin Date: Thu, 27 Oct 2016 12:45:55 -0700 Subject: [PATCH] Add VictorOps alert notification capability --- pkg/metrics/metrics.go | 3 +- pkg/services/alerting/notifiers/victorops.go | 94 +++++++++++++++++++ .../app/features/alerting/alert_tab_ctrl.ts | 1 + .../alerting/partials/notification_edit.html | 10 +- 4 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 pkg/services/alerting/notifiers/victorops.go diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 25fa43d8d2f..b83c63b0845 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -46,7 +46,7 @@ var ( M_Alerting_Notification_Sent_Email Counter M_Alerting_Notification_Sent_Webhook Counter M_Alerting_Notification_Sent_PagerDuty Counter - + M_Alerting_Notification_Sent_Victorops Counter // Timers M_DataSource_ProxyReq_Timer Timer @@ -110,6 +110,7 @@ func initMetricVars(settings *MetricSettings) { M_Alerting_Notification_Sent_Email = RegCounter("alerting.notifications_sent", "type", "email") M_Alerting_Notification_Sent_Webhook = RegCounter("alerting.notifications_sent", "type", "webhook") M_Alerting_Notification_Sent_PagerDuty = RegCounter("alerting.notifications_sent", "type", "pagerduty") + M_Alerting_Notification_Sent_Victorops = RegCounter("alerting.notifications_sent", "type", "victorops") // Timers M_DataSource_ProxyReq_Timer = RegTimer("api.dataproxy.request.all") diff --git a/pkg/services/alerting/notifiers/victorops.go b/pkg/services/alerting/notifiers/victorops.go new file mode 100644 index 00000000000..4d22de3ab9a --- /dev/null +++ b/pkg/services/alerting/notifiers/victorops.go @@ -0,0 +1,94 @@ +package notifiers + +import ( + "encoding/json" + "time" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/metrics" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting" + "github.com/grafana/grafana/pkg/setting" +) + +// AlertStateCritical - Victorops uses "CRITICAL" string to indicate "Alerting" state +const AlertStateCritical = "CRITICAL" + +func init() { + alerting.RegisterNotifier("victorops", NewVictoropsNotifier) +} + +// NewVictoropsNotifier creates an instance of VictoropsNotifier that +// handles posting notifications to Victorops REST API +func NewVictoropsNotifier(model *models.AlertNotification) (alerting.Notifier, error) { + url := model.Settings.Get("url").MustString() + if url == "" { + return nil, alerting.ValidationError{Reason: "Could not find victorops url property in settings"} + } + + return &VictoropsNotifier{ + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + URL: url, + log: log.New("alerting.notifier.victorops"), + }, nil +} + +// VictoropsNotifier defines URL property for Victorops REST API +// and handles notification process by formatting POST body according to +// Victorops specifications (http://victorops.force.com/knowledgebase/articles/Integration/Alert-Ingestion-API-Documentation/) +type VictoropsNotifier struct { + NotifierBase + URL string + log log.Logger +} + +// Notify sends notification to Victorops via POST to URL endpoint +func (this *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error { + this.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.Id, "notification", this.Name) + metrics.M_Alerting_Notification_Sent_Victorops.Inc(1) + + fields := make([]map[string]interface{}, 0) + fieldLimitCount := 4 + for index, evt := range evalContext.EvalMatches { + fields = append(fields, map[string]interface{}{ + "title": evt.Metric, + "value": evt.Value, + "short": true, + }) + if index > fieldLimitCount { + break + } + } + + if evalContext.Error != nil { + fields = append(fields, map[string]interface{}{ + "title": "Error message", + "value": evalContext.Error.Error(), + "short": false, + }) + } + + messageType := evalContext.Rule.State + if evalContext.Rule.State == models.AlertStateAlerting { // translate 'Alerting' to 'CRITICAL' (Victorops analog) + messageType = AlertStateCritical + } + + body := map[string]interface{}{ + "message_type": messageType, + "entity_id": evalContext.Rule.Name, + "timestamp": time.Now().Unix(), + "state_start_time": evalContext.StartTime.Unix(), + "state_message": evalContext.Rule.Message, + "monitoring_tool": "Grafana v" + setting.BuildVersion, + } + + data, _ := json.Marshal(&body) + cmd := &models.SendWebhook{Url: this.URL, Body: string(data)} + + if err := bus.Dispatch(cmd); err != nil { + this.log.Error("Failed to send victorops notification", "error", err, "webhook", this.Name) + } + + return nil +} diff --git a/public/app/features/alerting/alert_tab_ctrl.ts b/public/app/features/alerting/alert_tab_ctrl.ts index c882a6ac3bf..6573ff27ece 100644 --- a/public/app/features/alerting/alert_tab_ctrl.ts +++ b/public/app/features/alerting/alert_tab_ctrl.ts @@ -89,6 +89,7 @@ export class AlertTabCtrl { switch (type) { case "email": return "fa fa-envelope"; case "slack": return "fa fa-slack"; + case "victorops": return "fa fa-exclamation-triangle"; case "webhook": return "fa fa-cubes"; case "pagerduty": return "fa fa-bullhorn"; } diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index 42f1f553d71..f2563279695 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -19,7 +19,7 @@
Type
-
@@ -87,6 +87,14 @@ +
+

Victorops settings

+
+ Url + +
+
+

Email addresses