2021-05-19 08:27:41 -05:00
|
|
|
package channels
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"encoding/json"
|
2022-03-14 18:27:10 -05:00
|
|
|
"errors"
|
2021-05-19 08:27:41 -05:00
|
|
|
"fmt"
|
2022-06-21 03:06:00 -05:00
|
|
|
|
2021-05-19 08:27:41 -05:00
|
|
|
"net/url"
|
|
|
|
"strings"
|
|
|
|
|
|
|
|
"github.com/prometheus/alertmanager/template"
|
|
|
|
"github.com/prometheus/alertmanager/types"
|
2022-06-21 03:06:00 -05:00
|
|
|
"github.com/prometheus/common/model"
|
2022-06-07 12:54:23 -05:00
|
|
|
|
|
|
|
"github.com/grafana/grafana/pkg/infra/log"
|
|
|
|
"github.com/grafana/grafana/pkg/models"
|
2022-06-21 03:06:00 -05:00
|
|
|
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
|
2021-05-19 08:27:41 -05:00
|
|
|
)
|
|
|
|
|
2021-10-07 09:33:50 -05:00
|
|
|
// GetDecryptedValueFn is a function that returns the decrypted value of
|
|
|
|
// the given key. If the key is not present, then it returns the fallback value.
|
2021-11-04 11:47:21 -05:00
|
|
|
type GetDecryptedValueFn func(ctx context.Context, sjd map[string][]byte, key string, fallback string) string
|
2021-10-07 09:33:50 -05:00
|
|
|
|
2022-03-14 18:27:10 -05:00
|
|
|
type AlertmanagerConfig struct {
|
|
|
|
*NotificationChannelConfig
|
|
|
|
URLs []*url.URL
|
|
|
|
BasicAuthUser string
|
|
|
|
BasicAuthPassword string
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewAlertmanagerConfig(config *NotificationChannelConfig, fn GetDecryptedValueFn) (*AlertmanagerConfig, error) {
|
|
|
|
urlStr := config.Settings.Get("url").MustString()
|
2021-05-19 08:27:41 -05:00
|
|
|
if urlStr == "" {
|
2022-03-14 18:27:10 -05:00
|
|
|
return nil, errors.New("could not find url property in settings")
|
2021-05-19 08:27:41 -05:00
|
|
|
}
|
|
|
|
var urls []*url.URL
|
|
|
|
for _, uS := range strings.Split(urlStr, ",") {
|
|
|
|
uS = strings.TrimSpace(uS)
|
|
|
|
if uS == "" {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
uS = strings.TrimSuffix(uS, "/") + "/api/v1/alerts"
|
2022-03-14 18:27:10 -05:00
|
|
|
url, err := url.Parse(uS)
|
2021-05-19 08:27:41 -05:00
|
|
|
if err != nil {
|
2022-03-14 18:27:10 -05:00
|
|
|
return nil, fmt.Errorf("invalid url property in settings: %w", err)
|
2021-05-19 08:27:41 -05:00
|
|
|
}
|
2022-03-14 18:27:10 -05:00
|
|
|
urls = append(urls, url)
|
|
|
|
}
|
|
|
|
return &AlertmanagerConfig{
|
|
|
|
NotificationChannelConfig: config,
|
|
|
|
URLs: urls,
|
|
|
|
BasicAuthUser: config.Settings.Get("basicAuthUser").MustString(),
|
|
|
|
BasicAuthPassword: fn(context.Background(), config.SecureSettings, "basicAuthPassword", config.Settings.Get("basicAuthPassword").MustString()),
|
|
|
|
}, nil
|
|
|
|
}
|
2021-05-19 08:27:41 -05:00
|
|
|
|
2022-03-14 18:27:10 -05:00
|
|
|
func AlertmanagerFactory(fc FactoryConfig) (NotificationChannel, error) {
|
|
|
|
config, err := NewAlertmanagerConfig(fc.Config, fc.DecryptFunc)
|
|
|
|
if err != nil {
|
|
|
|
return nil, receiverInitError{
|
|
|
|
Reason: err.Error(),
|
|
|
|
Cfg: *fc.Config,
|
|
|
|
}
|
2021-05-19 08:27:41 -05:00
|
|
|
}
|
2022-06-21 03:06:00 -05:00
|
|
|
return NewAlertmanagerNotifier(config, fc.ImageStore, nil, fc.DecryptFunc), nil
|
2022-03-14 18:27:10 -05:00
|
|
|
}
|
2021-05-19 08:27:41 -05:00
|
|
|
|
2022-03-14 18:27:10 -05:00
|
|
|
// NewAlertmanagerNotifier returns a new Alertmanager notifier.
|
2022-06-21 03:06:00 -05:00
|
|
|
func NewAlertmanagerNotifier(config *AlertmanagerConfig, images ImageStore, _ *template.Template, fn GetDecryptedValueFn) *AlertmanagerNotifier {
|
2021-05-19 08:27:41 -05:00
|
|
|
return &AlertmanagerNotifier{
|
2021-10-22 04:11:06 -05:00
|
|
|
Base: NewBase(&models.AlertNotification{
|
2022-03-14 18:27:10 -05:00
|
|
|
Uid: config.UID,
|
|
|
|
Name: config.Name,
|
|
|
|
DisableResolveMessage: config.DisableResolveMessage,
|
|
|
|
Settings: config.Settings,
|
2021-05-19 08:27:41 -05:00
|
|
|
}),
|
2022-06-21 03:06:00 -05:00
|
|
|
images: images,
|
2022-03-14 18:27:10 -05:00
|
|
|
urls: config.URLs,
|
|
|
|
basicAuthUser: config.BasicAuthUser,
|
|
|
|
basicAuthPassword: config.BasicAuthPassword,
|
2021-05-19 08:27:41 -05:00
|
|
|
logger: log.New("alerting.notifier.prometheus-alertmanager"),
|
2022-03-14 18:27:10 -05:00
|
|
|
}
|
2021-05-19 08:27:41 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
// AlertmanagerNotifier sends alert notifications to the alert manager
|
|
|
|
type AlertmanagerNotifier struct {
|
2021-10-22 04:11:06 -05:00
|
|
|
*Base
|
2022-06-21 03:06:00 -05:00
|
|
|
images ImageStore
|
2021-05-19 08:27:41 -05:00
|
|
|
|
|
|
|
urls []*url.URL
|
|
|
|
basicAuthUser string
|
|
|
|
basicAuthPassword string
|
|
|
|
logger log.Logger
|
|
|
|
}
|
|
|
|
|
|
|
|
// Notify sends alert notifications to Alertmanager.
|
|
|
|
func (n *AlertmanagerNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
|
2022-06-07 12:54:23 -05:00
|
|
|
n.logger.Debug("sending Alertmanager alert", "alertmanager", n.Name)
|
2021-05-19 08:27:41 -05:00
|
|
|
if len(as) == 0 {
|
|
|
|
return true, nil
|
|
|
|
}
|
|
|
|
|
2022-06-21 03:06:00 -05:00
|
|
|
_ = withStoredImages(ctx, n.logger, n.images,
|
2022-06-30 09:27:57 -05:00
|
|
|
func(index int, image ngmodels.Image) error {
|
2022-06-21 03:06:00 -05:00
|
|
|
// If there is an image for this alert and the image has been uploaded
|
|
|
|
// to a public URL then include it as an annotation
|
2022-06-30 09:27:57 -05:00
|
|
|
if image.URL != "" {
|
2022-06-21 03:06:00 -05:00
|
|
|
as[index].Annotations["image"] = model.LabelValue(image.URL)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}, as...)
|
|
|
|
|
2021-05-19 08:27:41 -05:00
|
|
|
body, err := json.Marshal(as)
|
|
|
|
if err != nil {
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
|
2021-10-11 08:50:50 -05:00
|
|
|
var (
|
|
|
|
lastErr error
|
|
|
|
numErrs int
|
|
|
|
)
|
2021-05-19 08:27:41 -05:00
|
|
|
for _, u := range n.urls {
|
|
|
|
if _, err := sendHTTPRequest(ctx, u, httpCfg{
|
|
|
|
user: n.basicAuthUser,
|
|
|
|
password: n.basicAuthPassword,
|
|
|
|
body: body,
|
|
|
|
}, n.logger); err != nil {
|
2022-10-19 16:36:54 -05:00
|
|
|
n.logger.Warn("failed to send to Alertmanager", "error", err, "alertmanager", n.Name, "url", u.String())
|
2021-10-11 08:50:50 -05:00
|
|
|
lastErr = err
|
|
|
|
numErrs++
|
2021-05-19 08:27:41 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-10-11 08:50:50 -05:00
|
|
|
if numErrs == len(n.urls) {
|
2021-05-19 08:27:41 -05:00
|
|
|
// All attempts to send alerts have failed
|
2022-06-07 12:54:23 -05:00
|
|
|
n.logger.Warn("all attempts to send to Alertmanager failed", "alertmanager", n.Name)
|
2021-10-11 08:50:50 -05:00
|
|
|
return false, fmt.Errorf("failed to send alert to Alertmanager: %w", lastErr)
|
2021-05-19 08:27:41 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return true, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (n *AlertmanagerNotifier) SendResolved() bool {
|
|
|
|
return !n.GetDisableResolveMessage()
|
|
|
|
}
|