mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Alerting: Enable Alert rule tags to override PagerDuty Severity setting (#22736)
This commit is contained in:
Andrew Burian
GitHub
parent
aeb88015ab
commit
7d9b6a0dee
@ -110,7 +110,7 @@ Integration Key | Integration key for PagerDuty.
|
||||
Severity | Level for dynamic notifications, default is `critical`
|
||||
Auto resolve incidents | Resolve incidents in PagerDuty once the alert goes back to ok
|
||||
|
||||
**Note:** The tags `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys they will be transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools.
|
||||
**Note:** The tags `Severity`, `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level.
|
||||
|
||||
### Webhook
|
||||
|
||||
|
||||
@ -98,6 +98,7 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
|
||||
|
||||
// set default, override in following case switch if defined
|
||||
payloadJSON.Set("component", "Grafana")
|
||||
payloadJSON.Set("severity", pn.Severity)
|
||||
|
||||
for _, tag := range evalContext.Rule.AlertRuleTags {
|
||||
customData.Set(tag.Key, tag.Value)
|
||||
@ -110,6 +111,21 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
|
||||
payloadJSON.Set("class", tag.Value)
|
||||
case "component":
|
||||
payloadJSON.Set("component", tag.Value)
|
||||
case "severity":
|
||||
// Only set severity if it's one of the PD supported enum values
|
||||
// Info, Warning, Error, or Critical (case insensitive)
|
||||
switch sev := strings.ToLower(tag.Value); sev {
|
||||
case "info":
|
||||
fallthrough
|
||||
case "warning":
|
||||
fallthrough
|
||||
case "error":
|
||||
fallthrough
|
||||
case "critical":
|
||||
payloadJSON.Set("severity", sev)
|
||||
default:
|
||||
pn.log.Warn("Ignoring invalid severity tag", "severity", sev)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -122,7 +138,6 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
|
||||
if hostname, err := os.Hostname(); err == nil {
|
||||
payloadJSON.Set("source", hostname)
|
||||
}
|
||||
payloadJSON.Set("severity", pn.Severity)
|
||||
payloadJSON.Set("timestamp", time.Now())
|
||||
payloadJSON.Set("custom_details", customData)
|
||||
bodyJSON := simplejson.New()
|
||||
|
||||
@ -197,6 +197,7 @@ func TestPagerdutyNotifier(t *testing.T) {
|
||||
{Key: "group", Value: "aGroup"},
|
||||
{Key: "class", Value: "aClass"},
|
||||
{Key: "component", Value: "aComponent"},
|
||||
{Key: "severity", Value: "warning"},
|
||||
},
|
||||
})
|
||||
evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png"
|
||||
@ -224,6 +225,162 @@ func TestPagerdutyNotifier(t *testing.T) {
|
||||
"group": "aGroup",
|
||||
"class": "aClass",
|
||||
"component": "aComponent",
|
||||
"severity": "warning",
|
||||
"keyOnly": "",
|
||||
},
|
||||
"severity": "warning",
|
||||
"summary": "someRule - someMessage",
|
||||
"timestamp": "<<PRESENCE>>",
|
||||
"class": "aClass",
|
||||
"group": "aGroup",
|
||||
},
|
||||
"images": []interface{}{
|
||||
map[string]interface{}{
|
||||
"src": "http://somewhere.com/omg_dont_panic.png",
|
||||
},
|
||||
},
|
||||
"routing_key": "abcdefgh0123456789",
|
||||
}, payload.Interface(), cmp.Comparer(presenceComparer))
|
||||
So(diff, ShouldBeEmpty)
|
||||
})
|
||||
|
||||
Convey("should support multiple levels of severity", func() {
|
||||
json := `{
|
||||
"integrationKey": "abcdefgh0123456789",
|
||||
"autoResolve": false
|
||||
}`
|
||||
|
||||
settingsJSON, err := simplejson.NewJson([]byte(json))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
model := &models.AlertNotification{
|
||||
Name: "pagerduty_testing",
|
||||
Type: "pagerduty",
|
||||
Settings: settingsJSON,
|
||||
}
|
||||
|
||||
not, err := NewPagerdutyNotifier(model)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
pagerdutyNotifier := not.(*PagerdutyNotifier)
|
||||
|
||||
evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
|
||||
ID: 0,
|
||||
Name: "someRule",
|
||||
Message: "someMessage",
|
||||
State: models.AlertStateAlerting,
|
||||
AlertRuleTags: []*models.Tag{
|
||||
{Key: "keyOnly"},
|
||||
{Key: "group", Value: "aGroup"},
|
||||
{Key: "class", Value: "aClass"},
|
||||
{Key: "component", Value: "aComponent"},
|
||||
{Key: "severity", Value: "info"},
|
||||
},
|
||||
})
|
||||
evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png"
|
||||
evalContext.IsTestRun = true
|
||||
|
||||
payloadJSON, err := pagerdutyNotifier.buildEventPayload(evalContext)
|
||||
So(err, ShouldBeNil)
|
||||
payload, err := simplejson.NewJson(payloadJSON)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
diff := cmp.Diff(map[string]interface{}{
|
||||
"client": "Grafana",
|
||||
"client_url": "",
|
||||
"dedup_key": "alertId-0",
|
||||
"event_action": "trigger",
|
||||
"links": []interface{}{
|
||||
map[string]interface{}{
|
||||
"href": "",
|
||||
},
|
||||
},
|
||||
"payload": map[string]interface{}{
|
||||
"source": "<<PRESENCE>>",
|
||||
"component": "aComponent",
|
||||
"custom_details": map[string]interface{}{
|
||||
"group": "aGroup",
|
||||
"class": "aClass",
|
||||
"component": "aComponent",
|
||||
"severity": "info",
|
||||
"keyOnly": "",
|
||||
},
|
||||
"severity": "info",
|
||||
"summary": "someRule - someMessage",
|
||||
"timestamp": "<<PRESENCE>>",
|
||||
"class": "aClass",
|
||||
"group": "aGroup",
|
||||
},
|
||||
"images": []interface{}{
|
||||
map[string]interface{}{
|
||||
"src": "http://somewhere.com/omg_dont_panic.png",
|
||||
},
|
||||
},
|
||||
"routing_key": "abcdefgh0123456789",
|
||||
}, payload.Interface(), cmp.Comparer(presenceComparer))
|
||||
So(diff, ShouldBeEmpty)
|
||||
})
|
||||
|
||||
Convey("should ignore invalid severity for PD but keep the tag", func() {
|
||||
json := `{
|
||||
"integrationKey": "abcdefgh0123456789",
|
||||
"autoResolve": false,
|
||||
"severity": "critical"
|
||||
}`
|
||||
|
||||
settingsJSON, err := simplejson.NewJson([]byte(json))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
model := &models.AlertNotification{
|
||||
Name: "pagerduty_testing",
|
||||
Type: "pagerduty",
|
||||
Settings: settingsJSON,
|
||||
}
|
||||
|
||||
not, err := NewPagerdutyNotifier(model)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
pagerdutyNotifier := not.(*PagerdutyNotifier)
|
||||
|
||||
evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
|
||||
ID: 0,
|
||||
Name: "someRule",
|
||||
Message: "someMessage",
|
||||
State: models.AlertStateAlerting,
|
||||
AlertRuleTags: []*models.Tag{
|
||||
{Key: "keyOnly"},
|
||||
{Key: "group", Value: "aGroup"},
|
||||
{Key: "class", Value: "aClass"},
|
||||
{Key: "component", Value: "aComponent"},
|
||||
{Key: "severity", Value: "llama"},
|
||||
},
|
||||
})
|
||||
evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png"
|
||||
evalContext.IsTestRun = true
|
||||
|
||||
payloadJSON, err := pagerdutyNotifier.buildEventPayload(evalContext)
|
||||
So(err, ShouldBeNil)
|
||||
payload, err := simplejson.NewJson(payloadJSON)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
diff := cmp.Diff(map[string]interface{}{
|
||||
"client": "Grafana",
|
||||
"client_url": "",
|
||||
"dedup_key": "alertId-0",
|
||||
"event_action": "trigger",
|
||||
"links": []interface{}{
|
||||
map[string]interface{}{
|
||||
"href": "",
|
||||
},
|
||||
},
|
||||
"payload": map[string]interface{}{
|
||||
"source": "<<PRESENCE>>",
|
||||
"component": "aComponent",
|
||||
"custom_details": map[string]interface{}{
|
||||
"group": "aGroup",
|
||||
"class": "aClass",
|
||||
"component": "aComponent",
|
||||
"severity": "llama",
|
||||
"keyOnly": "",
|
||||
},
|
||||
"severity": "critical",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user