Alerting: Customise OK notification priorities for Pushover notifier (#30169)

* pushover: Customise OK notification priorities

* Apply suggestions from code review

* Fix build failure

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

* Update pkg/services/alerting/notifiers/pushover.go

Co-authored-by: Sofia Papagiannaki <papagian@users.noreply.github.com>

* Update docs

* Apply suggestions from code review

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Sofia Papagiannaki <papagian@users.noreply.github.com>
Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

docs/sources/administration/provisioning.md

@@ -421,13 +421,17 @@ The following sections detail the supported settings and secure settings for eac
 
 #### Alert notification `pushover`
 
-| Name     | Secure setting |
-| -------- | -------------- |
-| apiToken | yes            |
-| userKey  | yes            |
-| device   |                |
-| retry    |                |
-| expire   |                |
+| Name       | Secure setting |
+| --------   | -------------- |
+| apiToken   | yes            |
+| userKey    | yes            |
+| device     |                |
+| priority   |                |
+| okPriority |                |
+| retry      |                |
+| expire     |                |
+| sound      |                |
+| okSound    |                |
 
 #### Alert notification `slack`
 

docs/sources/alerting/notifications.md

@@ -62,7 +62,7 @@ Microsoft Teams | `teams` | yes, external only | no
 OpsGenie | `opsgenie` | yes, external only | yes
 [Pagerduty](#pagerduty) | `pagerduty` | yes, external only | yes
 Prometheus Alertmanager | `prometheus-alertmanager` | yes, external only | yes
-Pushover | `pushover` | yes | no
+[Pushover](#pushover) | `pushover` | yes | no
 Sensu | `sensu` | yes, external only | no
 [Sensu Go](#sensu-go) | `sensugo` | yes, external only | no
 [Slack](#slack) | `slack` | yes | no
@@ -130,6 +130,24 @@ Move any existing rules using `custom_details.myMetric` to `custom_details.queri
 This behavior will become the default in a future version of Grafana.
 
 > Using `dedup_key` tag will override Grafana generated `dedup_key` with a custom key.
+
+### Pushover
+
+To set up Pushover, you must provide a user key and an API token. Refer to [What is Pushover and how do I use it](https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it) for instructions on how to generate them.
+
+
+Setting | Description
+---------- | -----------
+API Token | Application token
+User key(s) | A comma-separated list of user keys
+Device(s) | A comma-separated list of devices
+Priority | The priority alerting nottifications are sent
+OK priority | The priority OK notifications are sent; if not set, then OK notifications are sent with the priority set for alerting notifications 
+Retry | How often (in seconds) the Pushover servers send the same notification to the user. (minimum 30 seconds)
+Expire | How many seconds your notification will continue to be retried for (maximum 86400 seconds)
+Alerting sound | The sound for alerting notifications
+OK sound | The sound for OK notifications
+
 ### Webhook
 
 The webhook notification is a simple way to send information about a state change over HTTP to a custom endpoint.

pkg/services/alerting/notifiers/pushover.go

@@ -91,6 +91,29 @@ func init() {
 		},
 	}
 
+	priorityOptions := []alerting.SelectOption{
+		{
+			Value: "2",
+			Label: "Emergency",
+		},
+		{
+			Value: "1",
+			Label: "High",
+		},
+		{
+			Value: "0",
+			Label: "Normal",
+		},
+		{
+			Value: "-1",
+			Label: "Low",
+		},
+		{
+			Value: "-2",
+			Label: "Lowest",
+		},
+	}
+
 	alerting.RegisterNotifier(&alerting.NotifierPlugin{
 		Type:        "pushover",
 		Name:        "Pushover",
@@ -124,53 +147,32 @@ func init() {
 				PropertyName: "device",
 			},
 			{
-				Label:   "Priority",
-				Element: alerting.ElementTypeSelect,
-				SelectOptions: []alerting.SelectOption{
-					{
-						Value: "2",
-						Label: "Emergency",
-					},
-					{
-						Value: "1",
-						Label: "High",
-					},
-					{
-						Value: "0",
-						Label: "Normal",
-					},
-					{
-						Value: "-1",
-						Label: "Low",
-					},
-					{
-						Value: "-2",
-						Label: "Lowest",
-					},
-				},
-				PropertyName: "priority",
+				Label:         "Alerting priority",
+				Element:       alerting.ElementTypeSelect,
+				SelectOptions: priorityOptions,
+				PropertyName:  "priority",
 			},
 			{
-				Label:        "Retry",
+				Label:         "OK priority",
+				Element:       alerting.ElementTypeSelect,
+				SelectOptions: priorityOptions,
+				PropertyName:  "okPriority",
+			},
+			{
+				Description:  "How often (in seconds) the Pushover servers will send the same alerting or OK notification to the user.",
+				Label:        "Retry (Only used for Emergency Priority)",
 				Element:      alerting.ElementTypeInput,
 				InputType:    alerting.InputTypeText,
 				Placeholder:  "minimum 30 seconds",
 				PropertyName: "retry",
-				ShowWhen: alerting.ShowWhen{
-					Field: "priority",
-					Is:    "2",
-				},
 			},
 			{
-				Label:        "Expire",
+				Description:  "How many seconds the alerting or OK notification will continue to be retried.",
+				Label:        "Expire (Only used for Emergency Priority)",
 				Element:      alerting.ElementTypeInput,
 				InputType:    alerting.InputTypeText,
 				Placeholder:  "maximum 86400 seconds",
 				PropertyName: "expire",
-				ShowWhen: alerting.ShowWhen{
-					Field: "priority",
-					Is:    "2",
-				},
 			},
 			{
 				Label:         "Alerting sound",
@@ -193,7 +195,14 @@ func NewPushoverNotifier(model *models.AlertNotification) (alerting.Notifier, er
 	userKey := model.DecryptedValue("userKey", model.Settings.Get("userKey").MustString())
 	APIToken := model.DecryptedValue("apiToken", model.Settings.Get("apiToken").MustString())
 	device := model.Settings.Get("device").MustString()
-	priority, _ := strconv.Atoi(model.Settings.Get("priority").MustString())
+	alertingPriority, err := strconv.Atoi(model.Settings.Get("priority").MustString("0")) // default Normal
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert alerting priority to integer: %w", err)
+	}
+	okPriority, err := strconv.Atoi(model.Settings.Get("okPriority").MustString("0")) // default Normal
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert OK priority to integer: %w", err)
+	}
 	retry, _ := strconv.Atoi(model.Settings.Get("retry").MustString())
 	expire, _ := strconv.Atoi(model.Settings.Get("expire").MustString())
 	alertingSound := model.Settings.Get("sound").MustString()
@@ -207,17 +216,18 @@ func NewPushoverNotifier(model *models.AlertNotification) (alerting.Notifier, er
 		return nil, alerting.ValidationError{Reason: "API token not given"}
 	}
 	return &PushoverNotifier{
-		NotifierBase:  NewNotifierBase(model),
-		UserKey:       userKey,
-		APIToken:      APIToken,
-		Priority:      priority,
-		Retry:         retry,
-		Expire:        expire,
-		Device:        device,
-		AlertingSound: alertingSound,
-		OkSound:       okSound,
-		Upload:        uploadImage,
-		log:           log.New("alerting.notifier.pushover"),
+		NotifierBase:     NewNotifierBase(model),
+		UserKey:          userKey,
+		APIToken:         APIToken,
+		AlertingPriority: alertingPriority,
+		OKPriority:       okPriority,
+		Retry:            retry,
+		Expire:           expire,
+		Device:           device,
+		AlertingSound:    alertingSound,
+		OKSound:          okSound,
+		Upload:           uploadImage,
+		log:              log.New("alerting.notifier.pushover"),
 	}, nil
 }
 
@@ -225,16 +235,17 @@ func NewPushoverNotifier(model *models.AlertNotification) (alerting.Notifier, er
 // alert notifications to Pushover
 type PushoverNotifier struct {
 	NotifierBase
-	UserKey       string
-	APIToken      string
-	Priority      int
-	Retry         int
-	Expire        int
-	Device        string
-	AlertingSound string
-	OkSound       string
-	Upload        bool
-	log           log.Logger
+	UserKey          string
+	APIToken         string
+	AlertingPriority int
+	OKPriority       int
+	Retry            int
+	Expire           int
+	Device           string
+	AlertingSound    string
+	OKSound          string
+	Upload           bool
+	log              log.Logger
 }
 
 // Notify sends a alert notification to Pushover
@@ -322,12 +333,16 @@ func (pn *PushoverNotifier) genPushoverBody(evalContext *alerting.EvalContext, m
 	}
 
 	// Add priority
-	err = w.WriteField("priority", strconv.Itoa(pn.Priority))
+	priority := pn.AlertingPriority
+	if evalContext.Rule.State == models.AlertStateOK {
+		priority = pn.OKPriority
+	}
+	err = w.WriteField("priority", strconv.Itoa(priority))
 	if err != nil {
 		return nil, b, err
 	}
 
-	if pn.Priority == 2 {
+	if priority == 2 {
 		err = w.WriteField("retry", strconv.Itoa(pn.Retry))
 		if err != nil {
 			return nil, b, err
@@ -350,7 +365,7 @@ func (pn *PushoverNotifier) genPushoverBody(evalContext *alerting.EvalContext, m
 	// Add sound
 	sound := pn.AlertingSound
 	if evalContext.Rule.State == models.AlertStateOK {
-		sound = pn.OkSound
+		sound = pn.OKSound
 	}
 	if sound != "default" {
 		err = w.WriteField("sound", sound)

pkg/services/alerting/notifiers/pushover_test.go

@@ -35,6 +35,7 @@ func TestPushoverNotifier(t *testing.T) {
 					"apiToken": "4SrUFQL4A5V5TQ1z5Pg9nxHXPXSTve",
 					"userKey": "tzNZYf36y0ohWwXo4XoUrB61rz1A4o",
 					"priority": "1",
+					"okPriority": "2",
 					"sound": "pushover",
 					"okSound": "magic"
 				}`
@@ -54,9 +55,10 @@ func TestPushoverNotifier(t *testing.T) {
 				So(pushoverNotifier.Type, ShouldEqual, "pushover")
 				So(pushoverNotifier.APIToken, ShouldEqual, "4SrUFQL4A5V5TQ1z5Pg9nxHXPXSTve")
 				So(pushoverNotifier.UserKey, ShouldEqual, "tzNZYf36y0ohWwXo4XoUrB61rz1A4o")
-				So(pushoverNotifier.Priority, ShouldEqual, 1)
+				So(pushoverNotifier.AlertingPriority, ShouldEqual, 1)
+				So(pushoverNotifier.OKPriority, ShouldEqual, 2)
 				So(pushoverNotifier.AlertingSound, ShouldEqual, "pushover")
-				So(pushoverNotifier.OkSound, ShouldEqual, "magic")
+				So(pushoverNotifier.OKSound, ShouldEqual, "magic")
 			})
 		})
 	})
@@ -67,7 +69,7 @@ func TestGenPushoverBody(t *testing.T) {
 		Convey("Given common sounds", func() {
 			sirenSound := "siren_sound_tst"
 			successSound := "success_sound_tst"
-			notifier := &PushoverNotifier{AlertingSound: sirenSound, OkSound: successSound}
+			notifier := &PushoverNotifier{AlertingSound: sirenSound, OKSound: successSound}
 
 			Convey("When alert is firing - should use siren sound", func() {
 				evalContext := alerting.NewEvalContext(context.Background(),