Alerting: fix UI element for PagerDuty's severity field configuration (#58927)

* make severity a regular text field
* add logs + fallback to critical if empty

pkg/services/ngalert/notifier/channels/pagerduty.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/prometheus/alertmanager/notify"
 	"github.com/prometheus/alertmanager/template"
@@ -21,9 +22,12 @@ import (
 const (
 	pagerDutyEventTrigger = "trigger"
 	pagerDutyEventResolve = "resolve"
+
+	defaultSeverity = "critical"
 )
 
 var (
+	knownSeverity        = map[string]struct{}{defaultSeverity: {}, "error": {}, "warning": {}, "info": {}}
 	PagerdutyEventAPIURL = "https://events.pagerduty.com/v2/enqueue"
 )
 
@@ -80,7 +84,7 @@ func newPagerdutyNotifier(fc FactoryConfig) (*PagerdutyNotifier, error) {
 		images: fc.ImageStore,
 		settings: pagerdutySettings{
 			Key:      key,
-			Severity: fc.Config.Settings.Get("severity").MustString("critical"),
+			Severity: fc.Config.Settings.Get("severity").MustString(defaultSeverity),
 			customDetails: map[string]string{
 				"firing":       `{{ template "__text_alert_list" .Alerts.Firing }}`,
 				"resolved":     `{{ template "__text_alert_list" .Alerts.Resolved }}`,
@@ -152,6 +156,12 @@ func (pn *PagerdutyNotifier) buildPagerdutyMessage(ctx context.Context, alerts m
 		details[k] = detail
 	}
 
+	severity := strings.ToLower(tmpl(pn.settings.Severity))
+	if _, ok := knownSeverity[severity]; !ok {
+		pn.log.Warn("Severity is not in the list of known values - using default severity", "actualSeverity", severity, "defaultSeverity", defaultSeverity)
+		severity = defaultSeverity
+	}
+
 	msg := &pagerDutyMessage{
 		Client:      "Grafana",
 		ClientURL:   pn.tmpl.ExternalURL.String(),
@@ -165,7 +175,7 @@ func (pn *PagerdutyNotifier) buildPagerdutyMessage(ctx context.Context, alerts m
 		Payload: pagerDutyPayload{
 			Component:     tmpl(pn.settings.Component),
 			Summary:       tmpl(pn.settings.Summary),
-			Severity:      tmpl(pn.settings.Severity),
+			Severity:      severity,
 			CustomDetails: details,
 			Class:         tmpl(pn.settings.Class),
 			Group:         tmpl(pn.settings.Group),

pkg/services/ngalert/notifier/channels/pagerduty_test.go

@@ -7,14 +7,14 @@ import (
 	"os"
 	"testing"
 
-	"github.com/grafana/grafana/pkg/components/simplejson"
-	"github.com/grafana/grafana/pkg/services/secrets/fakes"
-	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
-
 	"github.com/prometheus/alertmanager/notify"
 	"github.com/prometheus/alertmanager/types"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/services/secrets/fakes"
+	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
 )
 
 func TestPagerdutyNotifier(t *testing.T) {
@@ -69,7 +69,43 @@ func TestPagerdutyNotifier(t *testing.T) {
 				Links:     []pagerDutyLink{{HRef: "http://localhost", Text: "External URL"}},
 			},
 			expMsgError: nil,
-		}, {
+		},
+		{
+			name:     "should map unknown severity",
+			settings: `{"integrationKey": "abcdefgh0123456789", "severity": "{{ .CommonLabels.severity }}"}`,
+			alerts: []*types.Alert{
+				{
+					Alert: model.Alert{
+						Labels:      model.LabelSet{"alertname": "alert1", "lbl1": "val1", "severity": "invalid-severity"},
+						Annotations: model.LabelSet{"ann1": "annv1", "__dashboardUid__": "abcd", "__panelId__": "efgh"},
+					},
+				},
+			},
+			expMsg: &pagerDutyMessage{
+				RoutingKey:  "abcdefgh0123456789",
+				DedupKey:    "6e3538104c14b583da237e9693b76debbc17f0f8058ef20492e5853096cf8733",
+				EventAction: "trigger",
+				Payload: pagerDutyPayload{
+					Summary:   "[FIRING:1]  (val1 invalid-severity)",
+					Source:    hostname,
+					Severity:  defaultSeverity,
+					Class:     "default",
+					Component: "Grafana",
+					Group:     "default",
+					CustomDetails: map[string]string{
+						"firing":       "\nValue: [no value]\nLabels:\n - alertname = alert1\n - lbl1 = val1\n - severity = invalid-severity\nAnnotations:\n - ann1 = annv1\nSilence: http://localhost/alerting/silence/new?alertmanager=grafana&matcher=alertname%3Dalert1&matcher=lbl1%3Dval1&matcher=severity%3Dinvalid-severity\nDashboard: http://localhost/d/abcd\nPanel: http://localhost/d/abcd?viewPanel=efgh\n",
+						"num_firing":   "1",
+						"num_resolved": "0",
+						"resolved":     "",
+					},
+				},
+				Client:    "Grafana",
+				ClientURL: "http://localhost",
+				Links:     []pagerDutyLink{{HRef: "http://localhost", Text: "External URL"}},
+			},
+			expMsgError: nil,
+		},
+		{
 			name:     "Default config with one alert and custom summary",
 			settings: `{"integrationKey": "abcdefgh0123456789", "summary": "Alerts firing: {{ len .Alerts.Firing }}"}`,
 			alerts: []*types.Alert{

pkg/services/ngalert/notifier/channels_config/available_channels.go

@@ -239,26 +239,11 @@ func GetAvailableNotifiers() []*NotifierPlugin {
 					Secure:       true,
 				},
 				{
-					Label:   "Severity",
+					Label:        "Severity",
-					Element: ElementTypeSelect,
+					Element:      ElementTypeInput,
-					SelectOptions: []SelectOption{
+					InputType:    InputTypeText,
-						{
+					Placeholder:  "critical",
-							Value: "critical",
+					Description:  "Severity of the event. It must be critical, error, warning, info - otherwise, the default is set which is critical. You can use templates",
-							Label: "Critical",
-						},
-						{
-							Value: "error",
-							Label: "Error",
-						},
-						{
-							Value: "warning",
-							Label: "Warning",
-						},
-						{
-							Value: "info",
-							Label: "Info",
-						},
-					},
 					PropertyName: "severity",
 				},
 				{ // New in 8.0.