Alerting: Allow sending notification tags to Opsgenie as extra properties (#30332)

* Alerting: Opsgenie send tags as extra properties

Allow users to select where to send notification tags when alerting via
OpsGenie. Supports sending tags as key/value details, concatenated
strings in tags or both.

Users will be able to see their tags as key/values under extra
properties in an alert on the Opsgenie UI. These key/values can
then be used in the platform for routing, templating etc.

  * Configurable delivery to either tags, extra properties or both

  * Default to current behaviour (tags only)

  * Support both so users can transition from tags to details

Add docs and clean up references

* Alerting: Add additional arg for Opsgenie tests

The NewEvalContext function now requires a 'PluginRequestValidator' argument.
As our test does not use the validator we can specify 'nil' to satisfy the
function and allow our test to pass as expected.

* Alerting: Opsgenie doc fixes

Accept suggested changes for Opsgenie docs

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* Alerting: Opsgenie provisioning settings docs

Add the new setting to the provisioning docs

* Alerting: Opsgenie doc typo correction

Move the comma (,) out of the preformatting tags for the setting value

* Alerting: Opsgenie refactor send switches

Refactor the send switches to be methods on the OpsgenieNotiefier
itself. This also cleans up the method names so that the code reads
a bit more logically as:

if we should send details: send details
if we should send tags: send tags

This avoids the calling code needing to care about passing the state
and allows an engineer working in the `createAlert` function to focus
on the results of applying the logic instead.

* Alerting: Opsgenie docs rename note


Rename the note heading to match the number to more clearly link them.

* Alerting: Opsgenie use standard reference to note

Refer to the note below as per recommendation and standards.

Fixes #30331

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

docs/sources/administration/provisioning.md

@@ -541,6 +541,7 @@ The following sections detail the supported settings and secure settings for eac
 | apiUrl           |                |
 | autoClose        |                |
 | overridePriority |                |
+| sendTagsAs       |                |
 
 #### Alert notification `telegram`
 

docs/sources/alerting/notifications.md

@@ -59,7 +59,7 @@ Hipchat | `hipchat` | yes, external only | no
 [Kafka](#kafka) | `kafka` | yes, external only | no
 Line | `line` | yes, external only | no
 Microsoft Teams | `teams` | yes, external only | no
-OpsGenie | `opsgenie` | yes, external only | yes
+[Opsgenie](#opsgenie) | `opsgenie` | yes, external only | yes
 [Pagerduty](#pagerduty) | `pagerduty` | yes, external only | yes
 Prometheus Alertmanager | `prometheus-alertmanager` | yes, external only | yes
 [Pushover](#pushover) | `pushover` | yes | no
@@ -111,6 +111,19 @@ Token | If provided, Grafana will upload the generated image via Slack's file.up
 
 If you are using the token for a slack bot, then you have to invite the bot to the channel you want to send notifications and add the channel to the recipient field.
 
+### Opsgenie
+
+To setup Opsgenie you will need an API Key and the Alert API Url. These can be obtained by configuring a new [Grafana Integration](https://docs.opsgenie.com/docs/grafana-integration).
+
+Setting | Description
+--------|------------
+Alert API URL | The API URL for your Opsgenie instance. This will normally be either `https://api.opsgenie.com` or, for EU customers, `https://api.eu.opsgenie.com`.
+API Key | The API Key as provided by Opsgenie for your configured Grafana integration.
+Override priority | Configures the alert priority using the `og_priority` tag. The `og_priority` tag must have one of the following values: `P1`, `P2`, `P3`, `P4`, or `P5`. Default is `False`.
+Send notification tags as | Specify how you would like [Notification Tags]({{< relref "create-alerts.md/#notifications" >}}) delivered to Opsgenie. They can be delivered as `Tags`, `Extra Properties` or both. Default is Tags. See note below for more information.
+
+> **Note:** When notification tags are sent as `Tags` they are concatenated into a string with a `key:value` format. If you prefer to receive the notifications tags as key/values under Extra Properties in Opsgenie then change the `Send notification tags as` to either `Extra Properties` or `Tags & Extra Properties`.
+
 ### PagerDuty
 
 To set up PagerDuty, all you have to do is to provide an integration key.

pkg/services/alerting/notifiers/opsgenie.go

@@ -11,6 +11,12 @@ import (
 	"github.com/grafana/grafana/pkg/services/alerting"
 )
 
+const (
+	sendTags    = "tags"
+	sendDetails = "details"
+	sendBoth    = "both"
+)
+
 func init() {
 	alerting.RegisterNotifier(&alerting.NotifierPlugin{
 		Type:        "opsgenie",
@@ -47,11 +53,31 @@ func init() {
 				Description:  "Allow the alert priority to be set using the og_priority tag",
 				PropertyName: "overridePriority",
 			},
+			{
+				Label:   "Send notification tags as",
+				Element: alerting.ElementTypeSelect,
+				SelectOptions: []alerting.SelectOption{
+					{
+						Value: sendTags,
+						Label: "Tags",
+					},
+					{
+						Value: sendDetails,
+						Label: "Extra Properties",
+					},
+					{
+						Value: sendBoth,
+						Label: "Tags & Extra Properties",
+					},
+				},
+				Description:  "Send the notification tags to Opsgenie as either Extra Properties, Tags or both",
+				PropertyName: "sendTagsAs",
+			},
 		},
 	})
 }
 
-var (
+const (
 	opsgenieAlertURL = "https://api.opsgenie.com/v2/alerts"
 )
 
@@ -68,12 +94,20 @@ func NewOpsGenieNotifier(model *models.AlertNotification) (alerting.Notifier, er
 		apiURL = opsgenieAlertURL
 	}
 
+	sendTagsAs := model.Settings.Get("sendTagsAs").MustString(sendTags)
+	if sendTagsAs != sendTags && sendTagsAs != sendDetails && sendTagsAs != sendBoth {
+		return nil, alerting.ValidationError{
+			Reason: fmt.Sprintf("Invalid value for sendTagsAs: %q", sendTagsAs),
+		}
+	}
+
 	return &OpsGenieNotifier{
 		NotifierBase:     NewNotifierBase(model),
 		APIKey:           apiKey,
 		APIUrl:           apiURL,
 		AutoClose:        autoClose,
 		OverridePriority: overridePriority,
+		SendTagsAs:       sendTagsAs,
 		log:              log.New("alerting.notifier.opsgenie"),
 	}, nil
 }
@@ -86,6 +120,7 @@ type OpsGenieNotifier struct {
 	APIUrl           string
 	AutoClose        bool
 	OverridePriority bool
+	SendTagsAs       string
 	log              log.Logger
 }
 
@@ -131,14 +166,18 @@ func (on *OpsGenieNotifier) createAlert(evalContext *alerting.EvalContext) error
 		details.Set("image", evalContext.ImagePublicURL)
 	}
 
-	bodyJSON.Set("details", details)
-
 	tags := make([]string, 0)
 	for _, tag := range evalContext.Rule.AlertRuleTags {
-		if len(tag.Value) > 0 {
+		if on.sendDetails() {
-			tags = append(tags, fmt.Sprintf("%s:%s", tag.Key, tag.Value))
+			details.Set(tag.Key, tag.Value)
-		} else {
+		}
-			tags = append(tags, tag.Key)
+
+		if on.sendTags() {
+			if len(tag.Value) > 0 {
+				tags = append(tags, fmt.Sprintf("%s:%s", tag.Key, tag.Value))
+			} else {
+				tags = append(tags, tag.Key)
+			}
 		}
 		if tag.Key == "og_priority" {
 			if on.OverridePriority {
@@ -150,6 +189,7 @@ func (on *OpsGenieNotifier) createAlert(evalContext *alerting.EvalContext) error
 		}
 	}
 	bodyJSON.Set("tags", tags)
+	bodyJSON.Set("details", details)
 
 	body, _ := bodyJSON.MarshalJSON()
 
@@ -194,3 +234,11 @@ func (on *OpsGenieNotifier) closeAlert(evalContext *alerting.EvalContext) error
 
 	return nil
 }
+
+func (on *OpsGenieNotifier) sendDetails() bool {
+	return on.SendTagsAs == sendDetails || on.SendTagsAs == sendBoth
+}
+
+func (on *OpsGenieNotifier) sendTags() bool {
+	return on.SendTagsAs == sendTags || on.SendTagsAs == sendBoth
+}

pkg/services/alerting/notifiers/opsgenie_test.go

@@ -51,10 +51,30 @@ func TestOpsGenieNotifier(t *testing.T) {
 				So(opsgenieNotifier.Type, ShouldEqual, "opsgenie")
 				So(opsgenieNotifier.APIKey, ShouldEqual, "abcdefgh0123456789")
 			})
+		})
 
-			Convey("alert payload should include tag pairs in a ['key1:value1'] format when a value exists and in ['key2'] format when a value is absent", func() {
+		Convey("Handling notification tags", func() {
-				json := `
+			Convey("invalid sendTagsAs value should return error", func() {
-				{
+				json := `{
+          "apiKey": "abcdefgh0123456789",
+          "sendTagsAs": "not_a_valid_value"
+                                }`
+
+				settingsJSON, _ := simplejson.NewJson([]byte(json))
+				model := &models.AlertNotification{
+					Name:     "opsgenie_testing",
+					Type:     "opsgenie",
+					Settings: settingsJSON,
+				}
+
+				_, err := NewOpsGenieNotifier(model)
+				So(err, ShouldNotBeNil)
+				So(err, ShouldHaveSameTypeAs, alerting.ValidationError{})
+				So(err.Error(), ShouldEndWith, "Invalid value for sendTagsAs: \"not_a_valid_value\"")
+			})
+
+			Convey("alert payload should include tag pairs only as an array in the tags key when sendAsTags is not set", func() {
+				json := `{
           "apiKey": "abcdefgh0123456789"
 				}`
 
@@ -83,11 +103,13 @@ func TestOpsGenieNotifier(t *testing.T) {
 				}, &validations.OSSPluginRequestValidator{})
 				evalContext.IsTestRun = true
 
-				receivedTags := make([]string, 0)
+				tags := make([]string, 0)
+				details := make(map[string]interface{})
 				bus.AddHandlerCtx("alerting", func(ctx context.Context, cmd *models.SendWebhookSync) error {
 					bodyJSON, err := simplejson.NewJson([]byte(cmd.Body))
 					if err == nil {
-						receivedTags = bodyJSON.Get("tags").MustStringArray([]string{})
+						tags = bodyJSON.Get("tags").MustStringArray([]string{})
+						details = bodyJSON.Get("details").MustMap(map[string]interface{}{})
 					}
 					return err
 				})
@@ -96,7 +118,108 @@ func TestOpsGenieNotifier(t *testing.T) {
 
 				So(notifierErr, ShouldBeNil)
 				So(alertErr, ShouldBeNil)
-				So(receivedTags, ShouldResemble, []string{"keyOnly", "aKey:aValue"})
+				So(tags, ShouldResemble, []string{"keyOnly", "aKey:aValue"})
+				So(details, ShouldResemble, map[string]interface{}{"url": ""})
+			})
+
+			Convey("alert payload should include tag pairs only as a map in the details key when sendAsTags=details", func() {
+				json := `{
+          "apiKey": "abcdefgh0123456789",
+          "sendTagsAs": "details"
+				}`
+
+				tagPairs := []*models.Tag{
+					{Key: "keyOnly"},
+					{Key: "aKey", Value: "aValue"},
+				}
+
+				settingsJSON, _ := simplejson.NewJson([]byte(json))
+				model := &models.AlertNotification{
+					Name:     "opsgenie_testing",
+					Type:     "opsgenie",
+					Settings: settingsJSON,
+				}
+
+				notifier, notifierErr := NewOpsGenieNotifier(model) // unhandled error
+
+				opsgenieNotifier := notifier.(*OpsGenieNotifier)
+
+				evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
+					ID:            0,
+					Name:          "someRule",
+					Message:       "someMessage",
+					State:         models.AlertStateAlerting,
+					AlertRuleTags: tagPairs,
+				}, nil)
+				evalContext.IsTestRun = true
+
+				tags := make([]string, 0)
+				details := make(map[string]interface{})
+				bus.AddHandlerCtx("alerting", func(ctx context.Context, cmd *models.SendWebhookSync) error {
+					bodyJSON, err := simplejson.NewJson([]byte(cmd.Body))
+					if err == nil {
+						tags = bodyJSON.Get("tags").MustStringArray([]string{})
+						details = bodyJSON.Get("details").MustMap(map[string]interface{}{})
+					}
+					return err
+				})
+
+				alertErr := opsgenieNotifier.createAlert(evalContext)
+
+				So(notifierErr, ShouldBeNil)
+				So(alertErr, ShouldBeNil)
+				So(tags, ShouldResemble, []string{})
+				So(details, ShouldResemble, map[string]interface{}{"keyOnly": "", "aKey": "aValue", "url": ""})
+			})
+
+			Convey("alert payload should include tag pairs as both a map in the details key and an array in the tags key when sendAsTags=both", func() {
+				json := `{
+          "apiKey": "abcdefgh0123456789",
+          "sendTagsAs": "both"
+				}`
+
+				tagPairs := []*models.Tag{
+					{Key: "keyOnly"},
+					{Key: "aKey", Value: "aValue"},
+				}
+
+				settingsJSON, _ := simplejson.NewJson([]byte(json))
+				model := &models.AlertNotification{
+					Name:     "opsgenie_testing",
+					Type:     "opsgenie",
+					Settings: settingsJSON,
+				}
+
+				notifier, notifierErr := NewOpsGenieNotifier(model) // unhandled error
+
+				opsgenieNotifier := notifier.(*OpsGenieNotifier)
+
+				evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
+					ID:            0,
+					Name:          "someRule",
+					Message:       "someMessage",
+					State:         models.AlertStateAlerting,
+					AlertRuleTags: tagPairs,
+				}, nil)
+				evalContext.IsTestRun = true
+
+				tags := make([]string, 0)
+				details := make(map[string]interface{})
+				bus.AddHandlerCtx("alerting", func(ctx context.Context, cmd *models.SendWebhookSync) error {
+					bodyJSON, err := simplejson.NewJson([]byte(cmd.Body))
+					if err == nil {
+						tags = bodyJSON.Get("tags").MustStringArray([]string{})
+						details = bodyJSON.Get("details").MustMap(map[string]interface{}{})
+					}
+					return err
+				})
+
+				alertErr := opsgenieNotifier.createAlert(evalContext)
+
+				So(notifierErr, ShouldBeNil)
+				So(alertErr, ShouldBeNil)
+				So(tags, ShouldResemble, []string{"keyOnly", "aKey:aValue"})
+				So(details, ShouldResemble, map[string]interface{}{"keyOnly": "", "aKey": "aValue", "url": ""})
 			})
 		})
 	})