package notifiers import ( "context" "os" "strconv" "strings" "time" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/setting" ) func init() { alerting.RegisterNotifier(&alerting.NotifierPlugin{ Type: "pagerduty", Name: "PagerDuty", Description: "Sends notifications to PagerDuty", Heading: "PagerDuty settings", Factory: NewPagerdutyNotifier, Options: []alerting.NotifierOption{ { Label: "Integration Key", Element: alerting.ElementTypeInput, InputType: alerting.InputTypeText, Placeholder: "Pagerduty Integration Key", PropertyName: "integrationKey", Required: true, Secure: true, }, { Label: "Severity", Element: alerting.ElementTypeSelect, SelectOptions: []alerting.SelectOption{ { Value: "critical", Label: "Critical", }, { Value: "error", Label: "Error", }, { Value: "warning", Label: "Warning", }, { Value: "info", Label: "Info", }, }, PropertyName: "severity", }, { Label: "Auto resolve incidents", Element: alerting.ElementTypeCheckbox, Description: "Resolve incidents in pagerduty once the alert goes back to ok.", PropertyName: "autoResolve", }, { Label: "Include message in details", Element: alerting.ElementTypeCheckbox, Description: "Move the alert message from the PD summary into the custom details. This changes the custom details object and may break event rules you have configured", PropertyName: "messageInDetails", }, }, }) } var ( pagerdutyEventAPIURL = "https://events.pagerduty.com/v2/enqueue" ) // NewPagerdutyNotifier is the constructor for the PagerDuty notifier func NewPagerdutyNotifier(model *models.AlertNotification, fn alerting.GetDecryptedValueFn) (alerting.Notifier, error) { severity := model.Settings.Get("severity").MustString("critical") autoResolve := model.Settings.Get("autoResolve").MustBool(false) key := fn(context.Background(), model.SecureSettings, "integrationKey", model.Settings.Get("integrationKey").MustString(), setting.SecretKey) messageInDetails := model.Settings.Get("messageInDetails").MustBool(false) if key == "" { return nil, alerting.ValidationError{Reason: "Could not find integration key property in settings"} } return &PagerdutyNotifier{ NotifierBase: NewNotifierBase(model), Key: key, Severity: severity, AutoResolve: autoResolve, MessageInDetails: messageInDetails, log: log.New("alerting.notifier.pagerduty"), }, nil } // PagerdutyNotifier is responsible for sending // alert notifications to pagerduty type PagerdutyNotifier struct { NotifierBase Key string Severity string AutoResolve bool MessageInDetails bool log log.Logger } // buildEventPayload is responsible for building the event payload body for sending to Pagerduty v2 API func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext) ([]byte, error) { eventType := "trigger" if evalContext.Rule.State == models.AlertStateOK { eventType = "resolve" } customData := simplejson.New() customData.Set("state", evalContext.Rule.State) if pn.MessageInDetails { queries := make(map[string]interface{}) for _, evt := range evalContext.EvalMatches { queries[evt.Metric] = evt.Value } customData.Set("queries", queries) customData.Set("message", evalContext.Rule.Message) } else { for _, evt := range evalContext.EvalMatches { customData.Set(evt.Metric, evt.Value) } } pn.log.Info("Notifying Pagerduty", "event_type", eventType) payloadJSON := simplejson.New() // set default, override in following case switch if defined payloadJSON.Set("component", "Grafana") payloadJSON.Set("severity", pn.Severity) dedupKey := "alertId-" + strconv.FormatInt(evalContext.Rule.ID, 10) for _, tag := range evalContext.Rule.AlertRuleTags { // Override tags appropriately if they are in the PagerDuty v2 API switch strings.ToLower(tag.Key) { case "group": payloadJSON.Set("group", tag.Value) case "class": payloadJSON.Set("class", tag.Value) case "component": payloadJSON.Set("component", tag.Value) case "dedup_key": if len(tag.Value) > 254 { tag.Value = tag.Value[0:254] } dedupKey = tag.Value case "severity": // Only set severity if it's one of the PD supported enum values // Info, Warning, Error, or Critical (case insensitive) switch sev := strings.ToLower(tag.Value); sev { case "info": fallthrough case "warning": fallthrough case "error": fallthrough case "critical": payloadJSON.Set("severity", sev) default: pn.log.Warn("Ignoring invalid severity tag", "severity", sev) } } customData.Set(tag.Key, tag.Value) } var summary string if pn.MessageInDetails || evalContext.Rule.Message == "" { summary = evalContext.Rule.Name } else { summary = evalContext.Rule.Name + " - " + evalContext.Rule.Message } if len(summary) > 1024 { summary = summary[0:1024] } payloadJSON.Set("summary", summary) if hostname, err := os.Hostname(); err == nil { payloadJSON.Set("source", hostname) } payloadJSON.Set("timestamp", time.Now()) payloadJSON.Set("custom_details", customData) bodyJSON := simplejson.New() bodyJSON.Set("routing_key", pn.Key) bodyJSON.Set("event_action", eventType) bodyJSON.Set("dedup_key", dedupKey) bodyJSON.Set("payload", payloadJSON) ruleURL, err := evalContext.GetRuleURL() if err != nil { pn.log.Error("Failed get rule link", "error", err) return []byte{}, err } links := make([]interface{}, 1) linkJSON := simplejson.New() linkJSON.Set("href", ruleURL) bodyJSON.Set("client_url", ruleURL) bodyJSON.Set("client", "Grafana") links[0] = linkJSON bodyJSON.Set("links", links) if pn.NeedsImage() && evalContext.ImagePublicURL != "" { contexts := make([]interface{}, 1) imageJSON := simplejson.New() imageJSON.Set("src", evalContext.ImagePublicURL) contexts[0] = imageJSON bodyJSON.Set("images", contexts) } body, _ := bodyJSON.MarshalJSON() return body, nil } // Notify sends an alert notification to PagerDuty func (pn *PagerdutyNotifier) Notify(evalContext *alerting.EvalContext) error { if evalContext.Rule.State == models.AlertStateOK && !pn.AutoResolve { pn.log.Info("Not sending a trigger to Pagerduty", "state", evalContext.Rule.State, "auto resolve", pn.AutoResolve) return nil } body, err := pn.buildEventPayload(evalContext) if err != nil { pn.log.Error("Unable to build PagerDuty event payload", "error", err) return err } cmd := &models.SendWebhookSync{ Url: pagerdutyEventAPIURL, Body: string(body), HttpMethod: "POST", HttpHeader: map[string]string{ "Content-Type": "application/json", }, } if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { pn.log.Error("Failed to send notification to Pagerduty", "error", err, "body", string(body)) return err } return nil }