diff --git a/docs/sources/alerting/notifications.md b/docs/sources/alerting/notifications.md index bb50ed5ae80..98503351721 100644 --- a/docs/sources/alerting/notifications.md +++ b/docs/sources/alerting/notifications.md @@ -110,7 +110,7 @@ Integration Key | Integration key for PagerDuty. Severity | Level for dynamic notifications, default is `critical` Auto resolve incidents | Resolve incidents in PagerDuty once the alert goes back to ok -**Note:** The tags `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys they will be transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. +**Note:** The tags `Severity`, `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level. ### Webhook diff --git a/pkg/services/alerting/notifiers/pagerduty.go b/pkg/services/alerting/notifiers/pagerduty.go index 20a48d35b78..a2b5675d20f 100644 --- a/pkg/services/alerting/notifiers/pagerduty.go +++ b/pkg/services/alerting/notifiers/pagerduty.go @@ -98,6 +98,7 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext // set default, override in following case switch if defined payloadJSON.Set("component", "Grafana") + payloadJSON.Set("severity", pn.Severity) for _, tag := range evalContext.Rule.AlertRuleTags { customData.Set(tag.Key, tag.Value) @@ -110,6 +111,21 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext payloadJSON.Set("class", tag.Value) case "component": payloadJSON.Set("component", tag.Value) + case "severity": + // Only set severity if it's one of the PD supported enum values + // Info, Warning, Error, or Critical (case insensitive) + switch sev := strings.ToLower(tag.Value); sev { + case "info": + fallthrough + case "warning": + fallthrough + case "error": + fallthrough + case "critical": + payloadJSON.Set("severity", sev) + default: + pn.log.Warn("Ignoring invalid severity tag", "severity", sev) + } } } @@ -122,7 +138,6 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext if hostname, err := os.Hostname(); err == nil { payloadJSON.Set("source", hostname) } - payloadJSON.Set("severity", pn.Severity) payloadJSON.Set("timestamp", time.Now()) payloadJSON.Set("custom_details", customData) bodyJSON := simplejson.New() diff --git a/pkg/services/alerting/notifiers/pagerduty_test.go b/pkg/services/alerting/notifiers/pagerduty_test.go index 9e2bc7860ab..b6f2e2e20df 100644 --- a/pkg/services/alerting/notifiers/pagerduty_test.go +++ b/pkg/services/alerting/notifiers/pagerduty_test.go @@ -197,6 +197,7 @@ func TestPagerdutyNotifier(t *testing.T) { {Key: "group", Value: "aGroup"}, {Key: "class", Value: "aClass"}, {Key: "component", Value: "aComponent"}, + {Key: "severity", Value: "warning"}, }, }) evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png" @@ -224,6 +225,162 @@ func TestPagerdutyNotifier(t *testing.T) { "group": "aGroup", "class": "aClass", "component": "aComponent", + "severity": "warning", + "keyOnly": "", + }, + "severity": "warning", + "summary": "someRule - someMessage", + "timestamp": "<>", + "class": "aClass", + "group": "aGroup", + }, + "images": []interface{}{ + map[string]interface{}{ + "src": "http://somewhere.com/omg_dont_panic.png", + }, + }, + "routing_key": "abcdefgh0123456789", + }, payload.Interface(), cmp.Comparer(presenceComparer)) + So(diff, ShouldBeEmpty) + }) + + Convey("should support multiple levels of severity", func() { + json := `{ + "integrationKey": "abcdefgh0123456789", + "autoResolve": false + }` + + settingsJSON, err := simplejson.NewJson([]byte(json)) + So(err, ShouldBeNil) + + model := &models.AlertNotification{ + Name: "pagerduty_testing", + Type: "pagerduty", + Settings: settingsJSON, + } + + not, err := NewPagerdutyNotifier(model) + So(err, ShouldBeNil) + + pagerdutyNotifier := not.(*PagerdutyNotifier) + + evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{ + ID: 0, + Name: "someRule", + Message: "someMessage", + State: models.AlertStateAlerting, + AlertRuleTags: []*models.Tag{ + {Key: "keyOnly"}, + {Key: "group", Value: "aGroup"}, + {Key: "class", Value: "aClass"}, + {Key: "component", Value: "aComponent"}, + {Key: "severity", Value: "info"}, + }, + }) + evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png" + evalContext.IsTestRun = true + + payloadJSON, err := pagerdutyNotifier.buildEventPayload(evalContext) + So(err, ShouldBeNil) + payload, err := simplejson.NewJson(payloadJSON) + So(err, ShouldBeNil) + + diff := cmp.Diff(map[string]interface{}{ + "client": "Grafana", + "client_url": "", + "dedup_key": "alertId-0", + "event_action": "trigger", + "links": []interface{}{ + map[string]interface{}{ + "href": "", + }, + }, + "payload": map[string]interface{}{ + "source": "<>", + "component": "aComponent", + "custom_details": map[string]interface{}{ + "group": "aGroup", + "class": "aClass", + "component": "aComponent", + "severity": "info", + "keyOnly": "", + }, + "severity": "info", + "summary": "someRule - someMessage", + "timestamp": "<>", + "class": "aClass", + "group": "aGroup", + }, + "images": []interface{}{ + map[string]interface{}{ + "src": "http://somewhere.com/omg_dont_panic.png", + }, + }, + "routing_key": "abcdefgh0123456789", + }, payload.Interface(), cmp.Comparer(presenceComparer)) + So(diff, ShouldBeEmpty) + }) + + Convey("should ignore invalid severity for PD but keep the tag", func() { + json := `{ + "integrationKey": "abcdefgh0123456789", + "autoResolve": false, + "severity": "critical" + }` + + settingsJSON, err := simplejson.NewJson([]byte(json)) + So(err, ShouldBeNil) + + model := &models.AlertNotification{ + Name: "pagerduty_testing", + Type: "pagerduty", + Settings: settingsJSON, + } + + not, err := NewPagerdutyNotifier(model) + So(err, ShouldBeNil) + + pagerdutyNotifier := not.(*PagerdutyNotifier) + + evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{ + ID: 0, + Name: "someRule", + Message: "someMessage", + State: models.AlertStateAlerting, + AlertRuleTags: []*models.Tag{ + {Key: "keyOnly"}, + {Key: "group", Value: "aGroup"}, + {Key: "class", Value: "aClass"}, + {Key: "component", Value: "aComponent"}, + {Key: "severity", Value: "llama"}, + }, + }) + evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png" + evalContext.IsTestRun = true + + payloadJSON, err := pagerdutyNotifier.buildEventPayload(evalContext) + So(err, ShouldBeNil) + payload, err := simplejson.NewJson(payloadJSON) + So(err, ShouldBeNil) + + diff := cmp.Diff(map[string]interface{}{ + "client": "Grafana", + "client_url": "", + "dedup_key": "alertId-0", + "event_action": "trigger", + "links": []interface{}{ + map[string]interface{}{ + "href": "", + }, + }, + "payload": map[string]interface{}{ + "source": "<>", + "component": "aComponent", + "custom_details": map[string]interface{}{ + "group": "aGroup", + "class": "aClass", + "component": "aComponent", + "severity": "llama", "keyOnly": "", }, "severity": "critical",