Alerting: Add configurable severity support for PagerDuty notifier (#19425)

* Pagerduty notifier: configurable severity

Instead of hardcoding `critical` make it configurable per notification channel instance.

* fix html

* Add a test to ensure default severity is correct

* Notifications doc

* Add a non-default test

* Add err check on NewJson (all tests)

* Add default severity (critical) to AlertNotificationEditCtrl class

docs/sources/alerting/notifications.md

@@ -100,6 +100,7 @@ To set up PagerDuty, all you have to do is to provide an API key.
 Setting | Description
 ---------- | -----------
 Integration Key | Integration key for PagerDuty.
+Severity | Level for dynamic notifications, default is `critical`
 Auto resolve incidents | Resolve incidents in PagerDuty once the alert goes back to ok
 
 ### Webhook

pkg/services/alerting/notifiers/pagerduty.go

@@ -24,6 +24,16 @@ func init() {
         <span class="gf-form-label width-14">Integration Key</span>
         <input type="text" required class="gf-form-input max-width-22" ng-model="ctrl.model.settings.integrationKey" placeholder="Pagerduty Integration Key"></input>
       </div>
+      <div class="gf-form">
+        <span class="gf-form-label width-10">Severity</span>
+        <div class="gf-form-select-wrapper width-14">
+          <select
+            class="gf-form-input"
+            ng-model="ctrl.model.settings.severity"
+            ng-options="s for s in ['critical', 'error', 'warning', 'info']">
+          </select>
+        </div>
+      </div>
       <div class="gf-form">
         <gf-form-switch
            class="gf-form"
@@ -43,6 +53,7 @@ var (
 
 // NewPagerdutyNotifier is the constructor for the PagerDuty notifier
 func NewPagerdutyNotifier(model *models.AlertNotification) (alerting.Notifier, error) {
+	severity := model.Settings.Get("severity").MustString("critical")
 	autoResolve := model.Settings.Get("autoResolve").MustBool(false)
 	key := model.Settings.Get("integrationKey").MustString()
 	if key == "" {
@@ -52,6 +63,7 @@ func NewPagerdutyNotifier(model *models.AlertNotification) (alerting.Notifier, e
 	return &PagerdutyNotifier{
 		NotifierBase: NewNotifierBase(model),
 		Key:          key,
+		Severity:     severity,
 		AutoResolve:  autoResolve,
 		log:          log.New("alerting.notifier.pagerduty"),
 	}, nil
@@ -62,6 +74,7 @@ func NewPagerdutyNotifier(model *models.AlertNotification) (alerting.Notifier, e
 type PagerdutyNotifier struct {
 	NotifierBase
 	Key         string
+	Severity    string
 	AutoResolve bool
 	log         log.Logger
 }
@@ -96,7 +109,7 @@ func (pn *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error {
 	if hostname, err := os.Hostname(); err == nil {
 		payloadJSON.Set("source", hostname)
 	}
-	payloadJSON.Set("severity", "critical")
+	payloadJSON.Set("severity", pn.Severity)
 	payloadJSON.Set("timestamp", time.Now())
 	payloadJSON.Set("component", "Grafana")
 	payloadJSON.Set("custom_details", customData)

pkg/services/alerting/notifiers/pagerduty_test.go

@@ -14,7 +14,9 @@ func TestPagerdutyNotifier(t *testing.T) {
 			Convey("empty settings should return error", func() {
 				json := `{ }`
 
-				settingsJSON, _ := simplejson.NewJson([]byte(json))
+				settingsJSON, jerr := simplejson.NewJson([]byte(json))
+				So(jerr, ShouldBeNil)
+
 				model := &models.AlertNotification{
 					Name:     "pageduty_testing",
 					Type:     "pagerduty",
@@ -25,10 +27,12 @@ func TestPagerdutyNotifier(t *testing.T) {
 				So(err, ShouldNotBeNil)
 			})
 
-			Convey("auto resolve should default to false", func() {
-				json := `{ "integrationKey": "abcdefgh0123456789" }`
+			Convey("severity should override default", func() {
+				json := `{ "integrationKey": "abcdefgh0123456789", "severity": "info" }`
+
+				settingsJSON, jerr := simplejson.NewJson([]byte(json))
+				So(jerr, ShouldBeNil)
 
-				settingsJSON, _ := simplejson.NewJson([]byte(json))
 				model := &models.AlertNotification{
 					Name:     "pagerduty_testing",
 					Type:     "pagerduty",
@@ -42,6 +46,30 @@ func TestPagerdutyNotifier(t *testing.T) {
 				So(pagerdutyNotifier.Name, ShouldEqual, "pagerduty_testing")
 				So(pagerdutyNotifier.Type, ShouldEqual, "pagerduty")
 				So(pagerdutyNotifier.Key, ShouldEqual, "abcdefgh0123456789")
+				So(pagerdutyNotifier.Severity, ShouldEqual, "info")
+				So(pagerdutyNotifier.AutoResolve, ShouldBeFalse)
+			})
+
+			Convey("auto resolve and severity should have expected defaults", func() {
+				json := `{ "integrationKey": "abcdefgh0123456789" }`
+
+				settingsJSON, jerr := simplejson.NewJson([]byte(json))
+				So(jerr, ShouldBeNil)
+
+				model := &models.AlertNotification{
+					Name:     "pagerduty_testing",
+					Type:     "pagerduty",
+					Settings: settingsJSON,
+				}
+
+				not, err := NewPagerdutyNotifier(model)
+				pagerdutyNotifier := not.(*PagerdutyNotifier)
+
+				So(err, ShouldBeNil)
+				So(pagerdutyNotifier.Name, ShouldEqual, "pagerduty_testing")
+				So(pagerdutyNotifier.Type, ShouldEqual, "pagerduty")
+				So(pagerdutyNotifier.Key, ShouldEqual, "abcdefgh0123456789")
+				So(pagerdutyNotifier.Severity, ShouldEqual, "critical")
 				So(pagerdutyNotifier.AutoResolve, ShouldBeFalse)
 			})
 
@@ -52,7 +80,9 @@ func TestPagerdutyNotifier(t *testing.T) {
 					"autoResolve": false
 				}`
 
-				settingsJSON, _ := simplejson.NewJson([]byte(json))
+				settingsJSON, jerr := simplejson.NewJson([]byte(json))
+				So(jerr, ShouldBeNil)
+
 				model := &models.AlertNotification{
 					Name:     "pagerduty_testing",
 					Type:     "pagerduty",

public/app/features/alerting/NotificationsEditCtrl.ts

@@ -19,6 +19,7 @@ export class AlertNotificationEditCtrl {
     settings: {
       httpMethod: 'POST',
       autoResolve: true,
+      severity: 'critical',
       uploadImage: true,
     },
     isDefault: false,