Alerting: Enable Alert rule severity tag to override VictorOps Severity setting (#29392)

* add severity to victorops

* Update docs, add test

* Fix spelling and lint

* fix resolving not working

* Update docs/sources/alerting/notifications.md

* Update docs/sources/alerting/notifications.md

* Update docs/sources/alerting/notifications.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* Update docs/sources/alerting/notifications.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* remove table, update supported alert notifiers

* add docs

* 7.4->7.5

* fix

* Update docs/sources/alerting/notifications.md

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>
Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

docs/sources/alerting/notifications.md

@@ -68,7 +68,7 @@ Sensu | `sensu` | yes, external only | no
 [Slack](#slack) | `slack` | yes | no
 Telegram | `telegram` | yes | no
 Threema | `threema` | yes, external only | no
-VictorOps | `victorops` | yes, external only | no
+VictorOps | `victorops` | yes, external only | yes
 [Webhook](#webhook) | `webhook` | yes, external only | yes
 [Zenduty](#zenduty) | `webhook` | yes, external only | yes
 
@@ -133,6 +133,11 @@ This behavior will become the default in a future version of Grafana.
 
 > **Note:** The `state` tag overrides the current alert state inside the `custom_details` payload.
 
+### VictorOps
+
+To configure VictorOps, provide the URL from the Grafana Integration and substitute `$routing_key` with a valid key.
+
+> **Note:** The tag `Severity` has special meaning in the [VictorOps Incident Fields](https://help.victorops.com/knowledge-base/incident-fields-glossary/). If an alert panel defines this key, then it replaces the `message_type` in the root of the event sent to VictorOps.
 ### Pushover
 
 To set up Pushover, you must provide a user key and an API token. Refer to [What is Pushover and how do I use it](https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it) for instructions on how to generate them.

docs/sources/installation/upgrading.md

@@ -313,4 +313,10 @@ NOTE: Only snapshots created on Grafana 7.3 or later will use this column to sto
 
 The Grafana Docker images use the `root` group instead of the `grafana` group. This change can cause builds to break for users who extend the Grafana Docker image. Learn more about this change in the  [Docker migration instructions]({{< relref "docker/#migrate-to-v73-or-later">}})
 
+## Upgrading to v7.5
 
+### VictorOps Alert Notifier
+
+The VictorOps alert notifier now accepts a `severity` tag, in a similar vein to the PagerDuty alert notifier. The possible values are outlined in the [VictorOps docs](https://help.victorops.com/knowledge-base/incident-fields-glossary/).
+
+For example, if you want an alert to be `INFO`-level in VictorOps, create a tag `severity=info` (case-insensitive) in your alert.

pkg/services/alerting/notifiers/victorops.go

@@ -1,6 +1,7 @@
 package notifiers
 
 import (
+	"strings"
 	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
@@ -74,22 +75,35 @@ type VictoropsNotifier struct {
 	log             log.Logger
 }
 
-// Notify sends notification to Victorops via POST to URL endpoint
+func (vn *VictoropsNotifier) buildEventPayload(evalContext *alerting.EvalContext) (*simplejson.Json, error) {
-func (vn *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error {
-	vn.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.ID, "notification", vn.Name)
-
 	ruleURL, err := evalContext.GetRuleURL()
 	if err != nil {
 		vn.log.Error("Failed get rule link", "error", err)
-		return err
+		return nil, err
 	}
 
 	if evalContext.Rule.State == models.AlertStateOK && !vn.AutoResolve {
 		vn.log.Info("Not alerting VictorOps", "state", evalContext.Rule.State, "auto resolve", vn.AutoResolve)
-		return nil
+		return nil, nil
 	}
 
 	messageType := AlertStateCritical // Default to alerting and change based on state checks (Ensures string type)
+	for _, tag := range evalContext.Rule.AlertRuleTags {
+		if strings.ToLower(tag.Key) == "severity" {
+			// Only set severity if it's one of the PD supported enum values
+			// Info, Warning, Error, or Critical (case insensitive)
+			switch sev := strings.ToUpper(tag.Value); sev {
+			case "INFO":
+				fallthrough
+			case "WARNING":
+				fallthrough
+			case "CRITICAL":
+				messageType = sev
+			default:
+				vn.log.Warn("Ignoring invalid severity tag", "severity", sev)
+			}
+		}
+	}
 
 	if evalContext.Rule.State == models.AlertStateNoData { // translate 'NODATA' to set alert
 		messageType = vn.NoDataAlertType
@@ -127,6 +141,18 @@ func (vn *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error {
 		bodyJSON.Set("image_url", evalContext.ImagePublicURL)
 	}
 
+	return bodyJSON, nil
+}
+
+// Notify sends notification to Victorops via POST to URL endpoint
+func (vn *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error {
+	vn.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.ID, "notification", vn.Name)
+
+	bodyJSON, err := vn.buildEventPayload(evalContext)
+	if err != nil {
+		return err
+	}
+
 	data, _ := bodyJSON.MarshalJSON()
 	cmd := &models.SendWebhookSync{Url: vn.URL, Body: string(data)}
 

pkg/services/alerting/notifiers/victorops_test.go

@@ -1,13 +1,27 @@
 package notifiers
 
 import (
+	"context"
 	"testing"
 
+	"github.com/grafana/grafana/pkg/services/validations"
+
+	"github.com/google/go-cmp/cmp"
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting"
 	. "github.com/smartystreets/goconvey/convey"
 )
 
+func presenceComparerInt(a, b int64) bool {
+	if a == -1 {
+		return b != 0
+	}
+	if b == -1 {
+		return a != 0
+	}
+	return a == b
+}
 func TestVictoropsNotifier(t *testing.T) {
 	Convey("Victorops notifier tests", t, func() {
 		Convey("Parsing alert notification from settings", func() {
@@ -46,6 +60,103 @@ func TestVictoropsNotifier(t *testing.T) {
 				So(victoropsNotifier.Type, ShouldEqual, "victorops")
 				So(victoropsNotifier.URL, ShouldEqual, "http://google.com")
 			})
+
+			Convey("should return properly formatted event payload when using severity override tag", func() {
+				json := `
+				{
+					"url": "http://google.com"
+				}`
+
+				settingsJSON, err := simplejson.NewJson([]byte(json))
+				So(err, ShouldBeNil)
+
+				model := &models.AlertNotification{
+					Name:     "victorops_testing",
+					Type:     "victorops",
+					Settings: settingsJSON,
+				}
+
+				not, err := NewVictoropsNotifier(model)
+				So(err, ShouldBeNil)
+
+				victoropsNotifier := not.(*VictoropsNotifier)
+
+				evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
+					ID:      0,
+					Name:    "someRule",
+					Message: "someMessage",
+					State:   models.AlertStateAlerting,
+					AlertRuleTags: []*models.Tag{
+						{Key: "keyOnly"},
+						{Key: "severity", Value: "warning"},
+					},
+				}, &validations.OSSPluginRequestValidator{})
+				evalContext.IsTestRun = true
+
+				payload, err := victoropsNotifier.buildEventPayload(evalContext)
+				So(err, ShouldBeNil)
+
+				diff := cmp.Diff(map[string]interface{}{
+					"alert_url":           "",
+					"entity_display_name": "[Alerting] someRule",
+					"entity_id":           "someRule",
+					"message_type":        "WARNING",
+					"metrics":             map[string]interface{}{},
+					"monitoring_tool":     "Grafana v",
+					"state_message":       "someMessage",
+					"state_start_time":    int64(-1),
+					"timestamp":           int64(-1),
+				}, payload.Interface(), cmp.Comparer(presenceComparerInt))
+				So(diff, ShouldBeEmpty)
+			})
+			Convey("resolving with severity works properly", func() {
+				json := `
+				{
+					"url": "http://google.com"
+				}`
+
+				settingsJSON, err := simplejson.NewJson([]byte(json))
+				So(err, ShouldBeNil)
+
+				model := &models.AlertNotification{
+					Name:     "victorops_testing",
+					Type:     "victorops",
+					Settings: settingsJSON,
+				}
+
+				not, err := NewVictoropsNotifier(model)
+				So(err, ShouldBeNil)
+
+				victoropsNotifier := not.(*VictoropsNotifier)
+
+				evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
+					ID:      0,
+					Name:    "someRule",
+					Message: "someMessage",
+					State:   models.AlertStateOK,
+					AlertRuleTags: []*models.Tag{
+						{Key: "keyOnly"},
+						{Key: "severity", Value: "warning"},
+					},
+				}, &validations.OSSPluginRequestValidator{})
+				evalContext.IsTestRun = true
+
+				payload, err := victoropsNotifier.buildEventPayload(evalContext)
+				So(err, ShouldBeNil)
+
+				diff := cmp.Diff(map[string]interface{}{
+					"alert_url":           "",
+					"entity_display_name": "[OK] someRule",
+					"entity_id":           "someRule",
+					"message_type":        "RECOVERY",
+					"metrics":             map[string]interface{}{},
+					"monitoring_tool":     "Grafana v",
+					"state_message":       "someMessage",
+					"state_start_time":    int64(-1),
+					"timestamp":           int64(-1),
+				}, payload.Interface(), cmp.Comparer(presenceComparerInt))
+				So(diff, ShouldBeEmpty)
+			})
 		})
 	})
 }