From 7a9a52c317a435e6d861993db29f554c47429013 Mon Sep 17 00:00:00 2001 From: "Ottavio M. Hartman" Date: Thu, 18 Mar 2021 09:50:07 -0400 Subject: [PATCH] Alerting: Enable Alert rule `severity` tag to override VictorOps Severity setting (#29392) * add severity to victorops * Update docs, add test * Fix spelling and lint * fix resolving not working * Update docs/sources/alerting/notifications.md * Update docs/sources/alerting/notifications.md * Update docs/sources/alerting/notifications.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Update docs/sources/alerting/notifications.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * remove table, update supported alert notifiers * add docs * 7.4->7.5 * fix * Update docs/sources/alerting/notifications.md Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> --- docs/sources/alerting/notifications.md | 7 +- docs/sources/installation/upgrading.md | 6 + pkg/services/alerting/notifiers/victorops.go | 38 +++++- .../alerting/notifiers/victorops_test.go | 111 ++++++++++++++++++ 4 files changed, 155 insertions(+), 7 deletions(-) diff --git a/docs/sources/alerting/notifications.md b/docs/sources/alerting/notifications.md index 58e264ab549..90bd06b1d97 100644 --- a/docs/sources/alerting/notifications.md +++ b/docs/sources/alerting/notifications.md @@ -68,7 +68,7 @@ Sensu | `sensu` | yes, external only | no [Slack](#slack) | `slack` | yes | no Telegram | `telegram` | yes | no Threema | `threema` | yes, external only | no -VictorOps | `victorops` | yes, external only | no +VictorOps | `victorops` | yes, external only | yes [Webhook](#webhook) | `webhook` | yes, external only | yes [Zenduty](#zenduty) | `webhook` | yes, external only | yes @@ -133,6 +133,11 @@ This behavior will become the default in a future version of Grafana. > **Note:** The `state` tag overrides the current alert state inside the `custom_details` payload. +### VictorOps + +To configure VictorOps, provide the URL from the Grafana Integration and substitute `$routing_key` with a valid key. + +> **Note:** The tag `Severity` has special meaning in the [VictorOps Incident Fields](https://help.victorops.com/knowledge-base/incident-fields-glossary/). If an alert panel defines this key, then it replaces the `message_type` in the root of the event sent to VictorOps. ### Pushover To set up Pushover, you must provide a user key and an API token. Refer to [What is Pushover and how do I use it](https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it) for instructions on how to generate them. diff --git a/docs/sources/installation/upgrading.md b/docs/sources/installation/upgrading.md index 24fc9736248..a9af5bf46e7 100755 --- a/docs/sources/installation/upgrading.md +++ b/docs/sources/installation/upgrading.md @@ -313,4 +313,10 @@ NOTE: Only snapshots created on Grafana 7.3 or later will use this column to sto The Grafana Docker images use the `root` group instead of the `grafana` group. This change can cause builds to break for users who extend the Grafana Docker image. Learn more about this change in the [Docker migration instructions]({{< relref "docker/#migrate-to-v73-or-later">}}) +## Upgrading to v7.5 +### VictorOps Alert Notifier + +The VictorOps alert notifier now accepts a `severity` tag, in a similar vein to the PagerDuty alert notifier. The possible values are outlined in the [VictorOps docs](https://help.victorops.com/knowledge-base/incident-fields-glossary/). + +For example, if you want an alert to be `INFO`-level in VictorOps, create a tag `severity=info` (case-insensitive) in your alert. diff --git a/pkg/services/alerting/notifiers/victorops.go b/pkg/services/alerting/notifiers/victorops.go index ae87150406c..1d1111deb83 100644 --- a/pkg/services/alerting/notifiers/victorops.go +++ b/pkg/services/alerting/notifiers/victorops.go @@ -1,6 +1,7 @@ package notifiers import ( + "strings" "time" "github.com/grafana/grafana/pkg/bus" @@ -74,22 +75,35 @@ type VictoropsNotifier struct { log log.Logger } -// Notify sends notification to Victorops via POST to URL endpoint -func (vn *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error { - vn.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.ID, "notification", vn.Name) - +func (vn *VictoropsNotifier) buildEventPayload(evalContext *alerting.EvalContext) (*simplejson.Json, error) { ruleURL, err := evalContext.GetRuleURL() if err != nil { vn.log.Error("Failed get rule link", "error", err) - return err + return nil, err } if evalContext.Rule.State == models.AlertStateOK && !vn.AutoResolve { vn.log.Info("Not alerting VictorOps", "state", evalContext.Rule.State, "auto resolve", vn.AutoResolve) - return nil + return nil, nil } messageType := AlertStateCritical // Default to alerting and change based on state checks (Ensures string type) + for _, tag := range evalContext.Rule.AlertRuleTags { + if strings.ToLower(tag.Key) == "severity" { + // Only set severity if it's one of the PD supported enum values + // Info, Warning, Error, or Critical (case insensitive) + switch sev := strings.ToUpper(tag.Value); sev { + case "INFO": + fallthrough + case "WARNING": + fallthrough + case "CRITICAL": + messageType = sev + default: + vn.log.Warn("Ignoring invalid severity tag", "severity", sev) + } + } + } if evalContext.Rule.State == models.AlertStateNoData { // translate 'NODATA' to set alert messageType = vn.NoDataAlertType @@ -127,6 +141,18 @@ func (vn *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error { bodyJSON.Set("image_url", evalContext.ImagePublicURL) } + return bodyJSON, nil +} + +// Notify sends notification to Victorops via POST to URL endpoint +func (vn *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error { + vn.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.ID, "notification", vn.Name) + + bodyJSON, err := vn.buildEventPayload(evalContext) + if err != nil { + return err + } + data, _ := bodyJSON.MarshalJSON() cmd := &models.SendWebhookSync{Url: vn.URL, Body: string(data)} diff --git a/pkg/services/alerting/notifiers/victorops_test.go b/pkg/services/alerting/notifiers/victorops_test.go index 00b00f61389..f5669cb9d65 100644 --- a/pkg/services/alerting/notifiers/victorops_test.go +++ b/pkg/services/alerting/notifiers/victorops_test.go @@ -1,13 +1,27 @@ package notifiers import ( + "context" "testing" + "github.com/grafana/grafana/pkg/services/validations" + + "github.com/google/go-cmp/cmp" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting" . "github.com/smartystreets/goconvey/convey" ) +func presenceComparerInt(a, b int64) bool { + if a == -1 { + return b != 0 + } + if b == -1 { + return a != 0 + } + return a == b +} func TestVictoropsNotifier(t *testing.T) { Convey("Victorops notifier tests", t, func() { Convey("Parsing alert notification from settings", func() { @@ -46,6 +60,103 @@ func TestVictoropsNotifier(t *testing.T) { So(victoropsNotifier.Type, ShouldEqual, "victorops") So(victoropsNotifier.URL, ShouldEqual, "http://google.com") }) + + Convey("should return properly formatted event payload when using severity override tag", func() { + json := ` + { + "url": "http://google.com" + }` + + settingsJSON, err := simplejson.NewJson([]byte(json)) + So(err, ShouldBeNil) + + model := &models.AlertNotification{ + Name: "victorops_testing", + Type: "victorops", + Settings: settingsJSON, + } + + not, err := NewVictoropsNotifier(model) + So(err, ShouldBeNil) + + victoropsNotifier := not.(*VictoropsNotifier) + + evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{ + ID: 0, + Name: "someRule", + Message: "someMessage", + State: models.AlertStateAlerting, + AlertRuleTags: []*models.Tag{ + {Key: "keyOnly"}, + {Key: "severity", Value: "warning"}, + }, + }, &validations.OSSPluginRequestValidator{}) + evalContext.IsTestRun = true + + payload, err := victoropsNotifier.buildEventPayload(evalContext) + So(err, ShouldBeNil) + + diff := cmp.Diff(map[string]interface{}{ + "alert_url": "", + "entity_display_name": "[Alerting] someRule", + "entity_id": "someRule", + "message_type": "WARNING", + "metrics": map[string]interface{}{}, + "monitoring_tool": "Grafana v", + "state_message": "someMessage", + "state_start_time": int64(-1), + "timestamp": int64(-1), + }, payload.Interface(), cmp.Comparer(presenceComparerInt)) + So(diff, ShouldBeEmpty) + }) + Convey("resolving with severity works properly", func() { + json := ` + { + "url": "http://google.com" + }` + + settingsJSON, err := simplejson.NewJson([]byte(json)) + So(err, ShouldBeNil) + + model := &models.AlertNotification{ + Name: "victorops_testing", + Type: "victorops", + Settings: settingsJSON, + } + + not, err := NewVictoropsNotifier(model) + So(err, ShouldBeNil) + + victoropsNotifier := not.(*VictoropsNotifier) + + evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{ + ID: 0, + Name: "someRule", + Message: "someMessage", + State: models.AlertStateOK, + AlertRuleTags: []*models.Tag{ + {Key: "keyOnly"}, + {Key: "severity", Value: "warning"}, + }, + }, &validations.OSSPluginRequestValidator{}) + evalContext.IsTestRun = true + + payload, err := victoropsNotifier.buildEventPayload(evalContext) + So(err, ShouldBeNil) + + diff := cmp.Diff(map[string]interface{}{ + "alert_url": "", + "entity_display_name": "[OK] someRule", + "entity_id": "someRule", + "message_type": "RECOVERY", + "metrics": map[string]interface{}{}, + "monitoring_tool": "Grafana v", + "state_message": "someMessage", + "state_start_time": int64(-1), + "timestamp": int64(-1), + }, payload.Interface(), cmp.Comparer(presenceComparerInt)) + So(diff, ShouldBeEmpty) + }) }) }) }