From ff112f07e3f274e4e94b46de155914932b5757ea Mon Sep 17 00:00:00 2001 From: Sofia Papagiannaki Date: Tue, 18 May 2021 17:31:51 +0300 Subject: [PATCH] [Alerting]: Add Sensu Go integration with the alert manager (#34045) * [Alerting]: Add sensugo notification channel * Apply suggestions from code review Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> * Do not include labels with concatenated rule UID and names * Modifications after syncing with main Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> --- pkg/services/ngalert/notifier/alertmanager.go | 2 + .../ngalert/notifier/available_channels.go | 58 ++++++ .../ngalert/notifier/channels/sensugo.go | 169 ++++++++++++++++ .../ngalert/notifier/channels/sensugo_test.go | 186 ++++++++++++++++++ .../alerting/api_available_channel_test.go | 121 ++++++++++++ 5 files changed, 536 insertions(+) create mode 100644 pkg/services/ngalert/notifier/channels/sensugo.go create mode 100644 pkg/services/ngalert/notifier/channels/sensugo_test.go diff --git a/pkg/services/ngalert/notifier/alertmanager.go b/pkg/services/ngalert/notifier/alertmanager.go index 78b35bf6fac..5a12a43f742 100644 --- a/pkg/services/ngalert/notifier/alertmanager.go +++ b/pkg/services/ngalert/notifier/alertmanager.go @@ -422,6 +422,8 @@ func (am *Alertmanager) buildReceiverIntegrations(receiver *apimodels.PostableAp n, err = channels.NewDingDingNotifier(cfg, tmpl) case "webhook": n, err = channels.NewWebHookNotifier(cfg, tmpl) + case "sensugo": + n, err = channels.NewSensuGoNotifier(cfg, tmpl) default: return nil, fmt.Errorf("notifier %s is not supported", r.Type) } diff --git a/pkg/services/ngalert/notifier/available_channels.go b/pkg/services/ngalert/notifier/available_channels.go index 341d6a9f881..0086c513928 100644 --- a/pkg/services/ngalert/notifier/available_channels.go +++ b/pkg/services/ngalert/notifier/available_channels.go @@ -243,6 +243,64 @@ func GetAvailableNotifiers() []*alerting.NotifierPlugin { }, }, }, + { + Type: "sensugo", + Name: "Sensu Go", + Description: "Sends HTTP POST request to a Sensu Go API", + Heading: "Sensu Go Settings", + Options: []alerting.NotifierOption{ + { + Label: "Backend URL", + Element: alerting.ElementTypeInput, + InputType: alerting.InputTypeText, + Placeholder: "http://sensu-api.local:8080", + PropertyName: "url", + Required: true, + }, + { + Label: "API Key", + Element: alerting.ElementTypeInput, + InputType: alerting.InputTypePassword, + Description: "API key to auth to Sensu Go backend", + PropertyName: "apikey", + Required: true, + Secure: true, + }, + { + Label: "Proxy entity name", + Element: alerting.ElementTypeInput, + InputType: alerting.InputTypeText, + Placeholder: "default", + PropertyName: "entity", + }, + { + Label: "Check name", + Element: alerting.ElementTypeInput, + InputType: alerting.InputTypeText, + Placeholder: "default", + PropertyName: "check", + }, + { + Label: "Handler", + Element: alerting.ElementTypeInput, + InputType: alerting.InputTypeText, + PropertyName: "handler", + }, + { + Label: "Namespace", + Element: alerting.ElementTypeInput, + InputType: alerting.InputTypeText, + Placeholder: "default", + PropertyName: "namespace", + }, + { // New in 8.0. + Label: "Message", + Element: alerting.ElementTypeTextArea, + Placeholder: `{{ template "default.message" . }}`, + PropertyName: "message", + }, + }, + }, { Type: "teams", Name: "Microsoft Teams", diff --git a/pkg/services/ngalert/notifier/channels/sensugo.go b/pkg/services/ngalert/notifier/channels/sensugo.go new file mode 100644 index 00000000000..e17afd88b85 --- /dev/null +++ b/pkg/services/ngalert/notifier/channels/sensugo.go @@ -0,0 +1,169 @@ +package channels + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "path" + "strings" + "time" + + gokit_log "github.com/go-kit/kit/log" + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting" + old_notifiers "github.com/grafana/grafana/pkg/services/alerting/notifiers" + "github.com/prometheus/alertmanager/notify" + "github.com/prometheus/alertmanager/template" + "github.com/prometheus/alertmanager/types" + "github.com/prometheus/common/model" +) + +type SensuGoNotifier struct { + old_notifiers.NotifierBase + log log.Logger + tmpl *template.Template + + URL string + Entity string + Check string + Namespace string + Handler string + APIKey string + Message string +} + +// NewSensuGoNotifier is the constructor for the SensuGo notifier +func NewSensuGoNotifier(model *NotificationChannelConfig, t *template.Template) (*SensuGoNotifier, error) { + if model.Settings == nil { + return nil, alerting.ValidationError{Reason: "No settings supplied"} + } + + url := model.Settings.Get("url").MustString() + if url == "" { + return nil, alerting.ValidationError{Reason: "Could not find URL property in settings"} + } + + apikey := model.DecryptedValue("apikey", model.Settings.Get("apikey").MustString()) + if apikey == "" { + return nil, alerting.ValidationError{Reason: "Could not find the API key property in settings"} + } + + return &SensuGoNotifier{ + NotifierBase: old_notifiers.NewNotifierBase(&models.AlertNotification{ + Uid: model.UID, + Name: model.Name, + Type: model.Type, + DisableResolveMessage: model.DisableResolveMessage, + Settings: model.Settings, + SecureSettings: model.SecureSettings, + }), + URL: url, + Entity: model.Settings.Get("entity").MustString(), + Check: model.Settings.Get("check").MustString(), + Namespace: model.Settings.Get("namespace").MustString(), + Handler: model.Settings.Get("handler").MustString(), + APIKey: apikey, + Message: model.Settings.Get("message").MustString(`{{ template "default.message" .}}`), + log: log.New("alerting.notifier.sensugo"), + tmpl: t, + }, nil +} + +// Notify sends an alert notification to Sensu Go +func (sn *SensuGoNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) { + sn.log.Debug("Sending Sensu Go result") + + data := notify.GetTemplateData(ctx, sn.tmpl, as, gokit_log.NewNopLogger()) + var tmplErr error + tmpl := notify.TmplText(sn.tmpl, data, &tmplErr) + + // Sensu Go alerts require an entity and a check. We set it to the user-specified + // value (optional), else we fallback and use the grafana rule anme and ruleID. + entity := sn.Entity + if entity == "" { + entity = "default" + } + + check := sn.Check + if check == "" { + check = "default" + } + + alerts := types.Alerts(as...) + status := 0 + if alerts.Status() == model.AlertFiring { + // TODO figure out about NoData old state (we used to send status 1 in that case) + status = 2 + } + + namespace := sn.Namespace + if namespace == "" { + namespace = "default" + } + + var handlers []string + if sn.Handler != "" { + handlers = []string{sn.Handler} + } + + u, err := url.Parse(sn.tmpl.ExternalURL.String()) + if err != nil { + return false, fmt.Errorf("failed to parse external URL: %w", err) + } + u.Path = path.Join(u.Path, "/alerting/list") + ruleURL := u.String() + bodyMsgType := map[string]interface{}{ + "entity": map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": entity, + "namespace": namespace, + }, + }, + "check": map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": check, + "labels": map[string]string{ + "ruleURL": ruleURL, + }, + }, + "output": tmpl(sn.Message), + "issued": time.Now().Unix(), + "interval": 86400, + "status": status, + "handlers": handlers, + }, + "ruleUrl": ruleURL, + } + + if tmplErr != nil { + return false, fmt.Errorf("failed to template sensugo message: %w", tmplErr) + } + + body, err := json.Marshal(bodyMsgType) + if err != nil { + return false, err + } + + cmd := &models.SendWebhookSync{ + Url: fmt.Sprintf("%s/api/core/v2/namespaces/%s/events", strings.TrimSuffix(sn.URL, "/"), namespace), + Body: string(body), + HttpMethod: "POST", + HttpHeader: map[string]string{ + "Content-Type": "application/json", + "Authorization": fmt.Sprintf("Key %s", sn.APIKey), + }, + } + if err := bus.DispatchCtx(ctx, cmd); err != nil { + sn.log.Error("Failed to send Sensu Go event", "error", err, "sensugo", sn.Name) + return false, err + } + + return true, nil +} + +func (sn *SensuGoNotifier) SendResolved() bool { + return !sn.GetDisableResolveMessage() +} diff --git a/pkg/services/ngalert/notifier/channels/sensugo_test.go b/pkg/services/ngalert/notifier/channels/sensugo_test.go new file mode 100644 index 00000000000..29923882531 --- /dev/null +++ b/pkg/services/ngalert/notifier/channels/sensugo_test.go @@ -0,0 +1,186 @@ +package channels + +import ( + "context" + "encoding/json" + "errors" + "net/url" + "testing" + "time" + + "github.com/prometheus/alertmanager/notify" + "github.com/prometheus/alertmanager/types" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting" +) + +func TestSensuGoNotifier(t *testing.T) { + tmpl := templateForTests(t) + + externalURL, err := url.Parse("http://localhost") + require.NoError(t, err) + tmpl.ExternalURL = externalURL + + cases := []struct { + name string + settings string + alerts []*types.Alert + expMsg map[string]interface{} + expInitError error + expMsgError error + }{ + { + name: "Default config with one alert", + settings: `{"url": "http://sensu-api.local:8080", "apikey": ""}`, + alerts: []*types.Alert{ + { + Alert: model.Alert{ + Labels: model.LabelSet{"__alert_rule_uid__": "rule uid", "alertname": "alert1", "lbl1": "val1"}, + Annotations: model.LabelSet{"ann1": "annv1"}, + }, + }, + }, + expMsg: map[string]interface{}{ + "entity": map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "default", + "namespace": "default", + }, + }, + "check": map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "default", + "labels": map[string]string{ + "ruleURL": "http://localhost/alerting/list", + }, + }, + "output": "\n**Firing**\nLabels:\n - alertname = alert1\n - __alert_rule_uid__ = rule uid\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSource: \n\n\n\n\n", + "issued": time.Now().Unix(), + "interval": 86400, + "status": 2, + "handlers": nil, + }, + "ruleUrl": "http://localhost/alerting/list", + }, + expInitError: nil, + expMsgError: nil, + }, { + name: "Custom config with multiple alerts", + settings: `{ + "url": "http://sensu-api.local:8080", + "entity": "grafana_instance_01", + "check": "grafana_rule_0", + "namespace": "namespace", + "handler": "myhandler", + "apikey": "", + "message": "{{ len .Alerts.Firing }} alerts are firing, {{ len .Alerts.Resolved }} are resolved" + }`, + alerts: []*types.Alert{ + { + Alert: model.Alert{ + Labels: model.LabelSet{"__alert_rule_uid__": "rule uid", "alertname": "alert1", "lbl1": "val1"}, + Annotations: model.LabelSet{"ann1": "annv1"}, + }, + }, { + Alert: model.Alert{ + Labels: model.LabelSet{"alertname": "alert1", "lbl1": "val2"}, + Annotations: model.LabelSet{"ann1": "annv2"}, + }, + }, + }, + expMsg: map[string]interface{}{ + "entity": map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "grafana_instance_01", + "namespace": "namespace", + }, + }, + "check": map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "grafana_rule_0", + "labels": map[string]string{ + "ruleURL": "http://localhost/alerting/list", + }, + }, + "output": "2 alerts are firing, 0 are resolved", + "issued": time.Now().Unix(), + "interval": 86400, + "status": 2, + "handlers": []string{"myhandler"}, + }, + "ruleUrl": "http://localhost/alerting/list", + }, + expInitError: nil, + expMsgError: nil, + }, { + name: "Error in initing: missing URL", + settings: `{ + "apikey": "" + }`, + expInitError: alerting.ValidationError{Reason: "Could not find URL property in settings"}, + }, { + name: "Error in initing: missing API key", + settings: `{ + "url": "http://sensu-api.local:8080" + }`, + expInitError: alerting.ValidationError{Reason: "Could not find the API key property in settings"}, + }, { + name: "Error in building message", + settings: `{ + "url": "http://sensu-api.local:8080", + "apikey": "", + "message": "{{ .Status }" + }`, + expMsgError: errors.New("failed to template sensugo message: template: :1: unexpected \"}\" in operand"), + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + settingsJSON, err := simplejson.NewJson([]byte(c.settings)) + require.NoError(t, err) + + m := &NotificationChannelConfig{ + Name: "Sensu Go", + Type: "sensugo", + Settings: settingsJSON, + } + + sn, err := NewSensuGoNotifier(m, tmpl) + if c.expInitError != nil { + require.Error(t, err) + require.Equal(t, c.expInitError.Error(), err.Error()) + return + } + require.NoError(t, err) + + body := "" + bus.AddHandlerCtx("test", func(ctx context.Context, webhook *models.SendWebhookSync) error { + body = webhook.Body + return nil + }) + + ctx := notify.WithGroupKey(context.Background(), "alertname") + ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": ""}) + ok, err := sn.Notify(ctx, c.alerts...) + if c.expMsgError != nil { + require.False(t, ok) + require.Error(t, err) + require.Equal(t, c.expMsgError.Error(), err.Error()) + return + } + require.NoError(t, err) + require.True(t, ok) + + expBody, err := json.Marshal(c.expMsg) + require.NoError(t, err) + + require.JSONEq(t, string(expBody), body) + }) + } +} diff --git a/pkg/tests/api/alerting/api_available_channel_test.go b/pkg/tests/api/alerting/api_available_channel_test.go index 5ca9bd6fa3b..9228e5d1a5a 100644 --- a/pkg/tests/api/alerting/api_available_channel_test.go +++ b/pkg/tests/api/alerting/api_available_channel_test.go @@ -485,6 +485,127 @@ var expAvailableChannelJsonOutput = ` } ] }, + { + "type": "sensugo", + "name": "Sensu Go", + "description": "Sends HTTP POST request to a Sensu Go API", + "heading": "Sensu Go Settings", + "info": "", + "options": [ + { + "element": "input", + "inputType": "text", + "label": "Backend URL", + "description": "", + "placeholder": "http://sensu-api.local:8080", + "propertyName": "url", + "selectOptions": null, + "showWhen": { + "field": "", + "is": "" + }, + "required": true, + "validationRule": "", + "secure": false + }, + { + "element": "input", + "inputType": "password", + "label": "API Key", + "description": "API key to auth to Sensu Go backend", + "placeholder": "", + "propertyName": "apikey", + "selectOptions": null, + "showWhen": { + "field": "", + "is": "" + }, + "required": true, + "validationRule": "", + "secure": true + }, + { + "element": "input", + "inputType": "text", + "label": "Proxy entity name", + "description": "", + "placeholder": "default", + "propertyName": "entity", + "selectOptions": null, + "showWhen": { + "field": "", + "is": "" + }, + "required": false, + "validationRule": "", + "secure": false + }, + { + "element": "input", + "inputType": "text", + "label": "Check name", + "description": "", + "placeholder": "default", + "propertyName": "check", + "selectOptions": null, + "showWhen": { + "field": "", + "is": "" + }, + "required": false, + "validationRule": "", + "secure": false + }, + { + "element": "input", + "inputType": "text", + "label": "Handler", + "description": "", + "placeholder": "", + "propertyName": "handler", + "selectOptions": null, + "showWhen": { + "field": "", + "is": "" + }, + "required": false, + "validationRule": "", + "secure": false + }, + { + "element": "input", + "inputType": "text", + "label": "Namespace", + "description": "", + "placeholder": "default", + "propertyName": "namespace", + "selectOptions": null, + "showWhen": { + "field": "", + "is": "" + }, + "required": false, + "validationRule": "", + "secure": false + }, + { + "element": "textarea", + "inputType": "", + "label": "Message", + "description": "", + "placeholder": "{{ template \"default.message\" . }}", + "propertyName": "message", + "selectOptions": null, + "showWhen": { + "field": "", + "is": "" + }, + "required": false, + "validationRule": "", + "secure": false + } + ] + }, { "type": "teams", "name": "Microsoft Teams",