Alerting: Enable Alert rule severity tag to override VictorOps Severity setting (#29392)

* add severity to victorops

* Update docs, add test

* Fix spelling and lint

* fix resolving not working

* Update docs/sources/alerting/notifications.md

* Update docs/sources/alerting/notifications.md

* Update docs/sources/alerting/notifications.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* Update docs/sources/alerting/notifications.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* remove table, update supported alert notifiers

* add docs

* 7.4->7.5

* fix

* Update docs/sources/alerting/notifications.md

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>
Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

pkg/services/alerting/notifiers/victorops.go

@@ -1,6 +1,7 @@
 package notifiers
 
 import (
+	"strings"
 	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
@@ -74,22 +75,35 @@ type VictoropsNotifier struct {
 	log             log.Logger
 }
 
-// Notify sends notification to Victorops via POST to URL endpoint
-func (vn *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error {
-	vn.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.ID, "notification", vn.Name)
-
+func (vn *VictoropsNotifier) buildEventPayload(evalContext *alerting.EvalContext) (*simplejson.Json, error) {
 	ruleURL, err := evalContext.GetRuleURL()
 	if err != nil {
 		vn.log.Error("Failed get rule link", "error", err)
-		return err
+		return nil, err
 	}
 
 	if evalContext.Rule.State == models.AlertStateOK && !vn.AutoResolve {
 		vn.log.Info("Not alerting VictorOps", "state", evalContext.Rule.State, "auto resolve", vn.AutoResolve)
-		return nil
+		return nil, nil
 	}
 
 	messageType := AlertStateCritical // Default to alerting and change based on state checks (Ensures string type)
+	for _, tag := range evalContext.Rule.AlertRuleTags {
+		if strings.ToLower(tag.Key) == "severity" {
+			// Only set severity if it's one of the PD supported enum values
+			// Info, Warning, Error, or Critical (case insensitive)
+			switch sev := strings.ToUpper(tag.Value); sev {
+			case "INFO":
+				fallthrough
+			case "WARNING":
+				fallthrough
+			case "CRITICAL":
+				messageType = sev
+			default:
+				vn.log.Warn("Ignoring invalid severity tag", "severity", sev)
+			}
+		}
+	}
 
 	if evalContext.Rule.State == models.AlertStateNoData { // translate 'NODATA' to set alert
 		messageType = vn.NoDataAlertType
@@ -127,6 +141,18 @@ func (vn *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error {
 		bodyJSON.Set("image_url", evalContext.ImagePublicURL)
 	}
 
+	return bodyJSON, nil
+}
+
+// Notify sends notification to Victorops via POST to URL endpoint
+func (vn *VictoropsNotifier) Notify(evalContext *alerting.EvalContext) error {
+	vn.log.Info("Executing victorops notification", "ruleId", evalContext.Rule.ID, "notification", vn.Name)
+
+	bodyJSON, err := vn.buildEventPayload(evalContext)
+	if err != nil {
+		return err
+	}
+
 	data, _ := bodyJSON.MarshalJSON()
 	cmd := &models.SendWebhookSync{Url: vn.URL, Body: string(data)}
 

pkg/services/alerting/notifiers/victorops_test.go

@@ -1,13 +1,27 @@
 package notifiers
 
 import (
+	"context"
 	"testing"
 
+	"github.com/grafana/grafana/pkg/services/validations"
+
+	"github.com/google/go-cmp/cmp"
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting"
 	. "github.com/smartystreets/goconvey/convey"
 )
 
+func presenceComparerInt(a, b int64) bool {
+	if a == -1 {
+		return b != 0
+	}
+	if b == -1 {
+		return a != 0
+	}
+	return a == b
+}
 func TestVictoropsNotifier(t *testing.T) {
 	Convey("Victorops notifier tests", t, func() {
 		Convey("Parsing alert notification from settings", func() {
@@ -46,6 +60,103 @@ func TestVictoropsNotifier(t *testing.T) {
 				So(victoropsNotifier.Type, ShouldEqual, "victorops")
 				So(victoropsNotifier.URL, ShouldEqual, "http://google.com")
 			})
+
+			Convey("should return properly formatted event payload when using severity override tag", func() {
+				json := `
+				{
+					"url": "http://google.com"
+				}`
+
+				settingsJSON, err := simplejson.NewJson([]byte(json))
+				So(err, ShouldBeNil)
+
+				model := &models.AlertNotification{
+					Name:     "victorops_testing",
+					Type:     "victorops",
+					Settings: settingsJSON,
+				}
+
+				not, err := NewVictoropsNotifier(model)
+				So(err, ShouldBeNil)
+
+				victoropsNotifier := not.(*VictoropsNotifier)
+
+				evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
+					ID:      0,
+					Name:    "someRule",
+					Message: "someMessage",
+					State:   models.AlertStateAlerting,
+					AlertRuleTags: []*models.Tag{
+						{Key: "keyOnly"},
+						{Key: "severity", Value: "warning"},
+					},
+				}, &validations.OSSPluginRequestValidator{})
+				evalContext.IsTestRun = true
+
+				payload, err := victoropsNotifier.buildEventPayload(evalContext)
+				So(err, ShouldBeNil)
+
+				diff := cmp.Diff(map[string]interface{}{
+					"alert_url":           "",
+					"entity_display_name": "[Alerting] someRule",
+					"entity_id":           "someRule",
+					"message_type":        "WARNING",
+					"metrics":             map[string]interface{}{},
+					"monitoring_tool":     "Grafana v",
+					"state_message":       "someMessage",
+					"state_start_time":    int64(-1),
+					"timestamp":           int64(-1),
+				}, payload.Interface(), cmp.Comparer(presenceComparerInt))
+				So(diff, ShouldBeEmpty)
+			})
+			Convey("resolving with severity works properly", func() {
+				json := `
+				{
+					"url": "http://google.com"
+				}`
+
+				settingsJSON, err := simplejson.NewJson([]byte(json))
+				So(err, ShouldBeNil)
+
+				model := &models.AlertNotification{
+					Name:     "victorops_testing",
+					Type:     "victorops",
+					Settings: settingsJSON,
+				}
+
+				not, err := NewVictoropsNotifier(model)
+				So(err, ShouldBeNil)
+
+				victoropsNotifier := not.(*VictoropsNotifier)
+
+				evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
+					ID:      0,
+					Name:    "someRule",
+					Message: "someMessage",
+					State:   models.AlertStateOK,
+					AlertRuleTags: []*models.Tag{
+						{Key: "keyOnly"},
+						{Key: "severity", Value: "warning"},
+					},
+				}, &validations.OSSPluginRequestValidator{})
+				evalContext.IsTestRun = true
+
+				payload, err := victoropsNotifier.buildEventPayload(evalContext)
+				So(err, ShouldBeNil)
+
+				diff := cmp.Diff(map[string]interface{}{
+					"alert_url":           "",
+					"entity_display_name": "[OK] someRule",
+					"entity_id":           "someRule",
+					"message_type":        "RECOVERY",
+					"metrics":             map[string]interface{}{},
+					"monitoring_tool":     "Grafana v",
+					"state_message":       "someMessage",
+					"state_start_time":    int64(-1),
+					"timestamp":           int64(-1),
+				}, payload.Interface(), cmp.Comparer(presenceComparerInt))
+				So(diff, ShouldBeEmpty)
+			})
 		})
 	})
 }