grafana/pkg/services/ngalert/notifier/channels/pagerduty.go

package channels

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"

	"github.com/prometheus/alertmanager/notify"
	"github.com/prometheus/alertmanager/template"
	"github.com/prometheus/alertmanager/types"
	"github.com/prometheus/common/model"

	"github.com/grafana/grafana/pkg/infra/log"
	"github.com/grafana/grafana/pkg/models"
	ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
	"github.com/grafana/grafana/pkg/services/notifications"
)

const (
	pagerDutyEventTrigger = "trigger"
	pagerDutyEventResolve = "resolve"
)

var (
	PagerdutyEventAPIURL = "https://events.pagerduty.com/v2/enqueue"
)

// PagerdutyNotifier is responsible for sending
// alert notifications to pagerduty
type PagerdutyNotifier struct {
	*Base
	tmpl     *template.Template
	log      log.Logger
	ns       notifications.WebhookSender
	images   ImageStore
	settings pagerdutySettings
}

type pagerdutySettings struct {
	Key           string `json:"integrationKey,omitempty" yaml:"integrationKey,omitempty"`
	Severity      string `json:"severity,omitempty" yaml:"severity,omitempty"`
	customDetails map[string]string
	Class         string `json:"class,omitempty" yaml:"class,omitempty"`
	Component     string `json:"component,omitempty" yaml:"component,omitempty"`
	Group         string `json:"group,omitempty" yaml:"group,omitempty"`
	Summary       string `json:"summary,omitempty" yaml:"summary,omitempty"`
}

func PagerdutyFactory(fc FactoryConfig) (NotificationChannel, error) {
	pdn, err := newPagerdutyNotifier(fc)
	if err != nil {
		return nil, receiverInitError{
			Reason: err.Error(),
			Cfg:    *fc.Config,
		}
	}
	return pdn, nil
}

// NewPagerdutyNotifier is the constructor for the PagerDuty notifier
func newPagerdutyNotifier(fc FactoryConfig) (*PagerdutyNotifier, error) {
	key := fc.DecryptFunc(context.Background(), fc.Config.SecureSettings, "integrationKey", fc.Config.Settings.Get("integrationKey").MustString())
	if key == "" {
		return nil, errors.New("could not find integration key property in settings")
	}

	return &PagerdutyNotifier{
		Base: NewBase(&models.AlertNotification{
			Uid:                   fc.Config.UID,
			Name:                  fc.Config.Name,
			Type:                  fc.Config.Type,
			DisableResolveMessage: fc.Config.DisableResolveMessage,
			Settings:              fc.Config.Settings,
		}),
		tmpl:   fc.Template,
		log:    log.New("alerting.notifier." + fc.Config.Name),
		ns:     fc.NotificationService,
		images: fc.ImageStore,
		settings: pagerdutySettings{
			Key:      key,
			Severity: fc.Config.Settings.Get("severity").MustString("critical"),
			customDetails: map[string]string{
				"firing":       `{{ template "__text_alert_list" .Alerts.Firing }}`,
				"resolved":     `{{ template "__text_alert_list" .Alerts.Resolved }}`,
				"num_firing":   `{{ .Alerts.Firing | len }}`,
				"num_resolved": `{{ .Alerts.Resolved | len }}`,
			},
			Class:     fc.Config.Settings.Get("class").MustString("default"),
			Component: fc.Config.Settings.Get("component").MustString("Grafana"),
			Group:     fc.Config.Settings.Get("group").MustString("default"),
			Summary:   fc.Config.Settings.Get("summary").MustString(DefaultMessageTitleEmbed),
		},
	}, nil
}

// Notify sends an alert notification to PagerDuty
func (pn *PagerdutyNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
	alerts := types.Alerts(as...)
	if alerts.Status() == model.AlertResolved && !pn.SendResolved() {
		pn.log.Debug("not sending a trigger to Pagerduty", "status", alerts.Status(), "auto resolve", pn.SendResolved())
		return true, nil
	}

	msg, eventType, err := pn.buildPagerdutyMessage(ctx, alerts, as)
	if err != nil {
		return false, fmt.Errorf("build pagerduty message: %w", err)
	}

	body, err := json.Marshal(msg)
	if err != nil {
		return false, fmt.Errorf("marshal json: %w", err)
	}

	pn.log.Info("notifying Pagerduty", "event_type", eventType)
	cmd := &models.SendWebhookSync{
		Url:        PagerdutyEventAPIURL,
		Body:       string(body),
		HttpMethod: "POST",
		HttpHeader: map[string]string{
			"Content-Type": "application/json",
		},
	}
	if err := pn.ns.SendWebhookSync(ctx, cmd); err != nil {
		return false, fmt.Errorf("send notification to Pagerduty: %w", err)
	}

	return true, nil
}

func (pn *PagerdutyNotifier) buildPagerdutyMessage(ctx context.Context, alerts model.Alerts, as []*types.Alert) (*pagerDutyMessage, string, error) {
	key, err := notify.ExtractGroupKey(ctx)
	if err != nil {
		return nil, "", err
	}

	eventType := pagerDutyEventTrigger
	if alerts.Status() == model.AlertResolved {
		eventType = pagerDutyEventResolve
	}

	var tmplErr error
	tmpl, data := TmplText(ctx, pn.tmpl, as, pn.log, &tmplErr)

	details := make(map[string]string, len(pn.settings.customDetails))
	for k, v := range pn.settings.customDetails {
		detail, err := pn.tmpl.ExecuteTextString(v, data)
		if err != nil {
			return nil, "", fmt.Errorf("%q: failed to template %q: %w", k, v, err)
		}
		details[k] = detail
	}

	msg := &pagerDutyMessage{
		Client:      "Grafana",
		ClientURL:   pn.tmpl.ExternalURL.String(),
		RoutingKey:  pn.settings.Key,
		EventAction: eventType,
		DedupKey:    key.Hash(),
		Links: []pagerDutyLink{{
			HRef: pn.tmpl.ExternalURL.String(),
			Text: "External URL",
		}},
		Payload: pagerDutyPayload{
			Component:     tmpl(pn.settings.Component),
			Summary:       tmpl(pn.settings.Summary),
			Severity:      tmpl(pn.settings.Severity),
			CustomDetails: details,
			Class:         tmpl(pn.settings.Class),
			Group:         tmpl(pn.settings.Group),
		},
	}

	_ = withStoredImages(ctx, pn.log, pn.images,
		func(_ int, image ngmodels.Image) error {
			if len(image.URL) != 0 {
				msg.Images = append(msg.Images, pagerDutyImage{Src: image.URL})
			}

			return nil
		},
		as...)

	if len(msg.Payload.Summary) > 1024 {
		// This is the Pagerduty limit.
		msg.Payload.Summary = msg.Payload.Summary[:1021] + "..."
	}

	if hostname, err := os.Hostname(); err == nil {
		// TODO: should this be configured like in Prometheus AM?
		msg.Payload.Source = hostname
	}

	if tmplErr != nil {
		pn.log.Warn("failed to template PagerDuty message", "error", tmplErr.Error())
	}

	return msg, eventType, nil
}

func (pn *PagerdutyNotifier) SendResolved() bool {
	return !pn.GetDisableResolveMessage()
}

type pagerDutyMessage struct {
	RoutingKey  string           `json:"routing_key,omitempty"`
	ServiceKey  string           `json:"service_key,omitempty"`
	DedupKey    string           `json:"dedup_key,omitempty"`
	EventAction string           `json:"event_action"`
	Payload     pagerDutyPayload `json:"payload"`
	Client      string           `json:"client,omitempty"`
	ClientURL   string           `json:"client_url,omitempty"`
	Links       []pagerDutyLink  `json:"links,omitempty"`
	Images      []pagerDutyImage `json:"images,omitempty"`
}

type pagerDutyLink struct {
	HRef string `json:"href"`
	Text string `json:"text"`
}

type pagerDutyImage struct {
	Src string `json:"src"`
}

type pagerDutyPayload struct {
	Summary       string            `json:"summary"`
	Source        string            `json:"source"`
	Severity      string            `json:"severity"`
	Class         string            `json:"class,omitempty"`
	Component     string            `json:"component,omitempty"`
	Group         string            `json:"group,omitempty"`
	CustomDetails map[string]string `json:"custom_details,omitempty"`
}