diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bfecf44889b..3f7d6986440 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -194,6 +194,7 @@ /devenv/dev-dashboards-without-uid/ @grafana/dashboards-squad /devenv/dev-dashboards/ @grafana/dashboards-squad /devenv/docker/blocks/alert_webhook_listener/ @grafana/alerting-backend +/devenv/docker/blocks/caddy_tls/ @grafana/alerting-backend /devenv/docker/blocks/clickhouse/ @grafana/partner-datasources /devenv/docker/blocks/collectd/ @grafana/observability-metrics /devenv/docker/blocks/etcd @grafana/grafana-app-platform-squad diff --git a/devenv/docker/blocks/caddy_tls/README.md b/devenv/docker/blocks/caddy_tls/README.md new file mode 100644 index 00000000000..464f076ffcc --- /dev/null +++ b/devenv/docker/blocks/caddy_tls/README.md @@ -0,0 +1,33 @@ +# TLS Caddy Server + +Starts a [Caddy server](https://caddyserver.com/) with TLS configured. + +## Setup + +- Caddy is setup to run on port 2081, so when configuring the webhook receiver in Grafana Alerting you should use the +following the following URL: `https://localhost:2081` +- Also, Caddy is configured to use a self-signed certificate and to check the client certificate (`require_and_verify` mode) +- Caddy is setup to log requests and has debug mode enabled to make it easier to investigate possible issues + +## TLS Certificates + +If you want to configure a webhook contact point in Grafana Alerting with TLS, you need to provide a certificate and key. + +You can find them in `/etc/caddy` directory in the container: + +``` shell +docker exec devenv-caddy_tls-1 ls /etc/caddy/ +``` + +### CA Certificate + +``` shell +docker exec devenv-caddy_tls-1 cat /etc/caddy/ca.pem +``` + +### Client certificates + +``` shell +docker exec devenv-caddy_tls-1 cat /etc/caddy/client.pem +docker exec devenv-caddy_tls-1 cat /etc/caddy/client.key +``` diff --git a/devenv/docker/blocks/caddy_tls/build/Caddyfile b/devenv/docker/blocks/caddy_tls/build/Caddyfile new file mode 100644 index 00000000000..3674950f821 --- /dev/null +++ b/devenv/docker/blocks/caddy_tls/build/Caddyfile @@ -0,0 +1,14 @@ +{ + debug +} + +localhost:2081 { + log + tls /etc/caddy/server.pem /etc/caddy/server.key { + ca_root /etc/caddy/ca.pem + client_auth { + mode require_and_verify + trust_pool file /etc/caddy/client.pem /etc/caddy/ca.pem + } + } +} diff --git a/devenv/docker/blocks/caddy_tls/build/Dockerfile b/devenv/docker/blocks/caddy_tls/build/Dockerfile new file mode 100644 index 00000000000..ebe6d1b4f4a --- /dev/null +++ b/devenv/docker/blocks/caddy_tls/build/Dockerfile @@ -0,0 +1,12 @@ +FROM caddy:2.8.4-alpine + +WORKDIR /etc/caddy +EXPOSE 2081 + +COPY Caddyfile ./Caddyfile +COPY san.cnf ./san.cnf +COPY gen_certs.sh ./gen_certs.sh + +RUN apk update && apk upgrade --no-cache && apk add openssl + +RUN ./gen_certs.sh diff --git a/devenv/docker/blocks/caddy_tls/build/gen_certs.sh b/devenv/docker/blocks/caddy_tls/build/gen_certs.sh new file mode 100755 index 00000000000..b82d2b58abb --- /dev/null +++ b/devenv/docker/blocks/caddy_tls/build/gen_certs.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +DAYS_VALID=3650 + +# Create CA certificate +openssl genpkey -algorithm RSA -out ca.key +openssl req -new -x509 -days $DAYS_VALID -key ca.key -out ca.pem -subj "/CN=My CA" + +# Create server certificate +openssl genpkey -algorithm RSA -out server.key +openssl req -new -key server.key -out server.csr -subj "/CN=localhost" +openssl x509 -req -days $DAYS_VALID -in server.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out server.pem -extfile san.cnf -extensions v3_req + +# Create client key and certificate +openssl genpkey -algorithm RSA -out client.key +openssl req -new -key client.key -out client.csr -subj "/CN=Client" +openssl x509 -req -days $DAYS_VALID -in client.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out client.pem -extfile san.cnf -extensions v3_req diff --git a/devenv/docker/blocks/caddy_tls/build/san.cnf b/devenv/docker/blocks/caddy_tls/build/san.cnf new file mode 100644 index 00000000000..d7148ac8d4f --- /dev/null +++ b/devenv/docker/blocks/caddy_tls/build/san.cnf @@ -0,0 +1,7 @@ +[ v3_req ] +subjectAltName = @alt_names + +[ alt_names ] +DNS.1 = localhost +IP.1 = 127.0.0.1 +IP.2 = ::1 diff --git a/devenv/docker/blocks/caddy_tls/docker-compose.yaml b/devenv/docker/blocks/caddy_tls/docker-compose.yaml new file mode 100644 index 00000000000..1c83b88ca1e --- /dev/null +++ b/devenv/docker/blocks/caddy_tls/docker-compose.yaml @@ -0,0 +1,5 @@ + caddy_tls: + build: + context: docker/blocks/caddy_tls/build + ports: + - "2081:2081" diff --git a/docs/sources/administration/provisioning/index.md b/docs/sources/administration/provisioning/index.md index a3681a643c9..4c1cb9df1d4 100644 --- a/docs/sources/administration/provisioning/index.md +++ b/docs/sources/administration/provisioning/index.md @@ -614,12 +614,22 @@ The following sections detail the supported settings and secure settings for eac #### Alert notification `webhook` -| Name | Secure setting | -| ---------- | -------------- | -| url | | -| httpMethod | | -| username | | -| password | yes | +| Name | Secure setting | +| ----------- | -------------- | +| url | | +| http_method | | +| username | | +| password | yes | +| tls_config | | + +##### TLS config + +| Name | Secure setting | +| ------------------ | -------------- | +| insecureSkipVerify | | +| clientCertificate | yes | +| clientKey | yes | +| caCertificate | yes | #### Alert notification `googlechat` diff --git a/docs/sources/alerting/set-up/provision-alerting-resources/file-provisioning/index.md b/docs/sources/alerting/set-up/provision-alerting-resources/file-provisioning/index.md index 11d92dc2243..28e6afd35a1 100644 --- a/docs/sources/alerting/set-up/provision-alerting-resources/file-provisioning/index.md +++ b/docs/sources/alerting/set-up/provision-alerting-resources/file-provisioning/index.md @@ -616,6 +616,16 @@ settings: authorization_credentials: abc123 # maxAlerts: '10' + # + tlsConfig: + # + insecureSkipVerify: false + # + clientCertificate: certificate in PEM format + # + clientKey: key in PEM format + # + caCertificate: CA certificate in PEM format ``` {{< /collapse >}} diff --git a/go.mod b/go.mod index 6138c6c3f49..fdde201defa 100644 --- a/go.mod +++ b/go.mod @@ -72,7 +72,7 @@ require ( github.com/googleapis/gax-go/v2 v2.13.0 // @grafana/grafana-backend-group github.com/gorilla/mux v1.8.1 // @grafana/grafana-backend-group github.com/gorilla/websocket v1.5.0 // @grafana/grafana-app-platform-squad - github.com/grafana/alerting v0.0.0-20241010165806-807ddf183724 // @grafana/alerting-backend + github.com/grafana/alerting v0.0.0-20241021123319-be61d61f71e7 // @grafana/alerting-backend github.com/grafana/authlib v0.0.0-20241018103850-afc1195d8240 // @grafana/identity-access-team github.com/grafana/authlib/claims v0.0.0-20241018085709-130ad686d80e // @grafana/identity-access-team github.com/grafana/codejen v0.0.3 // @grafana/dataviz-squad diff --git a/go.sum b/go.sum index 5152e1b75ca..3605874c827 100644 --- a/go.sum +++ b/go.sum @@ -2243,8 +2243,8 @@ github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grafana/alerting v0.0.0-20241010165806-807ddf183724 h1:u+ZM5TLkdeEoSWXgYWxc4XRfPHhXpR63MyHXJxbBLrc= -github.com/grafana/alerting v0.0.0-20241010165806-807ddf183724/go.mod h1:QsnoKX/iYZxA4Cv+H+wC7uxutBD8qi8ZW5UJvD2TYmU= +github.com/grafana/alerting v0.0.0-20241021123319-be61d61f71e7 h1:lsM/QscEX+ZDIJm48ynQscH+msETyGYV6ug8L4f2DtM= +github.com/grafana/alerting v0.0.0-20241021123319-be61d61f71e7/go.mod h1:QsnoKX/iYZxA4Cv+H+wC7uxutBD8qi8ZW5UJvD2TYmU= github.com/grafana/authlib v0.0.0-20241018103850-afc1195d8240 h1:bBn6sCbBjxjYlvs5JAIGHQSOs8xbDEBWbezxarA/DDo= github.com/grafana/authlib v0.0.0-20241018103850-afc1195d8240/go.mod h1:RKqhn8E5PY2k5Xo6X8FHFgP45/qt9qqfAY7YYJ2mtB8= github.com/grafana/authlib/claims v0.0.0-20241018085709-130ad686d80e h1:I0sSXcqdt/ttiOJ/BVhpfa2q/xAyWSweQwaypGmvLss= diff --git a/pkg/services/ngalert/api/tooling/definitions/contact_points.go b/pkg/services/ngalert/api/tooling/definitions/contact_points.go index 4aec132e837..bb38195c780 100644 --- a/pkg/services/ngalert/api/tooling/definitions/contact_points.go +++ b/pkg/services/ngalert/api/tooling/definitions/contact_points.go @@ -289,14 +289,15 @@ type WebhookIntegration struct { URL string `json:"url" yaml:"url" hcl:"url"` - HTTPMethod *string `json:"httpMethod,omitempty" yaml:"httpMethod,omitempty" hcl:"http_method"` - MaxAlerts *int64 `json:"maxAlerts,omitempty" yaml:"maxAlerts,omitempty" hcl:"max_alerts"` - AuthorizationScheme *string `json:"authorization_scheme,omitempty" yaml:"authorization_scheme,omitempty" hcl:"authorization_scheme"` - AuthorizationCredentials *Secret `json:"authorization_credentials,omitempty" yaml:"authorization_credentials,omitempty" hcl:"authorization_credentials"` - User *string `json:"username,omitempty" yaml:"username,omitempty" hcl:"basic_auth_user"` - Password *Secret `json:"password,omitempty" yaml:"password,omitempty" hcl:"basic_auth_password"` - Title *string `json:"title,omitempty" yaml:"title,omitempty" hcl:"title"` - Message *string `json:"message,omitempty" yaml:"message,omitempty" hcl:"message"` + HTTPMethod *string `json:"httpMethod,omitempty" yaml:"httpMethod,omitempty" hcl:"http_method"` + MaxAlerts *int64 `json:"maxAlerts,omitempty" yaml:"maxAlerts,omitempty" hcl:"max_alerts"` + AuthorizationScheme *string `json:"authorization_scheme,omitempty" yaml:"authorization_scheme,omitempty" hcl:"authorization_scheme"` + AuthorizationCredentials *Secret `json:"authorization_credentials,omitempty" yaml:"authorization_credentials,omitempty" hcl:"authorization_credentials"` + User *string `json:"username,omitempty" yaml:"username,omitempty" hcl:"basic_auth_user"` + Password *Secret `json:"password,omitempty" yaml:"password,omitempty" hcl:"basic_auth_password"` + Title *string `json:"title,omitempty" yaml:"title,omitempty" hcl:"title"` + Message *string `json:"message,omitempty" yaml:"message,omitempty" hcl:"message"` + TLSConfig *TLSConfig `json:"tlsConfig,omitempty" yaml:"tlsConfig,omitempty" hcl:"tlsConfig,block"` } type WecomIntegration struct { diff --git a/pkg/services/ngalert/notifier/channels_config/available_channels.go b/pkg/services/ngalert/notifier/channels_config/available_channels.go index ffe7b757edc..dac5e7e9f4f 100644 --- a/pkg/services/ngalert/notifier/channels_config/available_channels.go +++ b/pkg/services/ngalert/notifier/channels_config/available_channels.go @@ -964,6 +964,48 @@ func GetAvailableNotifiers() []*NotifierPlugin { PropertyName: "message", Placeholder: alertingTemplates.DefaultMessageEmbed, }, + { + Label: "TLS", + PropertyName: "tlsConfig", + Description: "TLS configuration options", + Element: ElementTypeSubform, + SubformOptions: []NotifierOption{ + { + Label: "Disable certificate verification", + Element: ElementTypeCheckbox, + Description: "Do not verify the server's certificate chain and host name.", + PropertyName: "insecureSkipVerify", + Required: false, + }, + { + Label: "CA Certificate", + Element: ElementTypeTextArea, + Description: "Certificate in PEM format to use when verifying the server's certificate chain.", + InputType: InputTypeText, + PropertyName: "caCertificate", + Required: false, + Secure: true, + }, + { + Label: "Client Certificate", + Element: ElementTypeTextArea, + Description: "Client certificate in PEM format to use when connecting to the server.", + InputType: InputTypeText, + PropertyName: "clientCertificate", + Required: false, + Secure: true, + }, + { + Label: "Client Key", + Element: ElementTypeTextArea, + Description: "Client key in PEM format to use when connecting to the server.", + InputType: InputTypeText, + PropertyName: "clientKey", + Required: false, + Secure: true, + }, + }, + }, }, }, { diff --git a/pkg/services/ngalert/notifier/channels_config/available_channels_test.go b/pkg/services/ngalert/notifier/channels_config/available_channels_test.go index 2171b2c17fe..5240c177873 100644 --- a/pkg/services/ngalert/notifier/channels_config/available_channels_test.go +++ b/pkg/services/ngalert/notifier/channels_config/available_channels_test.go @@ -22,7 +22,7 @@ func TestGetSecretKeysForContactPointType(t *testing.T) { {receiverType: "sensugo", expectedSecretFields: []string{"apikey"}}, {receiverType: "teams", expectedSecretFields: []string{}}, {receiverType: "telegram", expectedSecretFields: []string{"bottoken"}}, - {receiverType: "webhook", expectedSecretFields: []string{"password", "authorization_credentials"}}, + {receiverType: "webhook", expectedSecretFields: []string{"password", "authorization_credentials", "tlsConfig.caCertificate", "tlsConfig.clientCertificate", "tlsConfig.clientKey"}}, {receiverType: "wecom", expectedSecretFields: []string{"url", "secret"}}, {receiverType: "prometheus-alertmanager", expectedSecretFields: []string{"basicAuthPassword"}}, {receiverType: "discord", expectedSecretFields: []string{"url"}}, diff --git a/pkg/services/ngalert/notifier/sender.go b/pkg/services/ngalert/notifier/sender.go index 55677c987d2..be2f58b523d 100644 --- a/pkg/services/ngalert/notifier/sender.go +++ b/pkg/services/ngalert/notifier/sender.go @@ -22,6 +22,7 @@ func (s sender) SendWebhook(ctx context.Context, cmd *receivers.SendWebhookSetti HttpHeader: cmd.HTTPHeader, ContentType: cmd.ContentType, Validation: cmd.Validation, + TLSConfig: cmd.TLSConfig, }) } diff --git a/pkg/services/notifications/models.go b/pkg/services/notifications/models.go index 289b2566c27..364c6e6d262 100644 --- a/pkg/services/notifications/models.go +++ b/pkg/services/notifications/models.go @@ -1,6 +1,7 @@ package notifications import ( + "crypto/tls" "errors" "github.com/grafana/grafana/pkg/services/user" @@ -42,6 +43,7 @@ type SendWebhookSync struct { HttpHeader map[string]string ContentType string Validation func(body []byte, statusCode int) error + TLSConfig *tls.Config } type SendResetPasswordEmailCommand struct { diff --git a/pkg/services/notifications/notifications.go b/pkg/services/notifications/notifications.go index b69b43f01c8..2be85628090 100644 --- a/pkg/services/notifications/notifications.go +++ b/pkg/services/notifications/notifications.go @@ -120,7 +120,6 @@ func (ns *NotificationService) Run(ctx context.Context) error { select { case webhook := <-ns.webhookQueue: err := ns.sendWebRequestSync(context.Background(), webhook) - if err != nil { ns.log.Error("Failed to send webrequest ", "error", err) } @@ -155,6 +154,7 @@ func (ns *NotificationService) SendWebhookSync(ctx context.Context, cmd *SendWeb HttpMethod: cmd.HttpMethod, HttpHeader: cmd.HttpHeader, ContentType: cmd.ContentType, + TLSConfig: cmd.TLSConfig, Validation: cmd.Validation, }) } @@ -211,7 +211,6 @@ func (ns *NotificationService) SendEmailCommandHandlerSync(ctx context.Context, Subject: cmd.Subject, ReplyTo: cmd.ReplyTo, }) - if err != nil { return err } @@ -222,7 +221,6 @@ func (ns *NotificationService) SendEmailCommandHandlerSync(ctx context.Context, func (ns *NotificationService) SendEmailCommandHandler(ctx context.Context, cmd *SendEmailCommand) error { message, err := ns.buildEmailMessage(cmd) - if err != nil { return err } @@ -304,7 +302,6 @@ func (ns *NotificationService) signUpStartedHandler(ctx context.Context, evt *ev "SignUpUrl": setting.ToAbsUrl(fmt.Sprintf("signup/?email=%s&code=%s", url.QueryEscape(evt.Email), url.QueryEscape(evt.Code))), }, }) - if err != nil { return err } diff --git a/pkg/services/notifications/testing.go b/pkg/services/notifications/testing.go index f632bef5038..082c321801e 100644 --- a/pkg/services/notifications/testing.go +++ b/pkg/services/notifications/testing.go @@ -33,11 +33,3 @@ func NewFakeDisconnectedMailer() *FakeDisconnectedMailer { func (fdm *FakeDisconnectedMailer) Send(ctx context.Context, messages ...*Message) (int, error) { return 0, fmt.Errorf("connect: connection refused") } - -// NetClient is used to export original in test. -var NetClient = &netClient - -// SetWebhookClient is used to mock in test. -func SetWebhookClient(client WebhookClient) { - netClient = client -} diff --git a/pkg/services/notifications/webhook.go b/pkg/services/notifications/webhook.go index 7fbaf59eaad..be92ff76b40 100644 --- a/pkg/services/notifications/webhook.go +++ b/pkg/services/notifications/webhook.go @@ -7,10 +7,10 @@ import ( "errors" "fmt" "io" - "net" "net/http" "net/url" - "time" + + alertingReceivers "github.com/grafana/alerting/receivers" "github.com/grafana/grafana/pkg/util" ) @@ -23,6 +23,7 @@ type Webhook struct { HttpMethod string HttpHeader map[string]string ContentType string + TLSConfig *tls.Config // Validation is a function that will validate the response body and statusCode of the webhook. Any returned error will cause the webhook request to be considered failed. // This can be useful when a webhook service communicates failures in creative ways, such as using the response body instead of the status code. @@ -34,21 +35,6 @@ type WebhookClient interface { Do(req *http.Request) (*http.Response, error) } -var netTransport = &http.Transport{ - TLSClientConfig: &tls.Config{ - Renegotiation: tls.RenegotiateFreelyAsClient, - }, - Proxy: http.ProxyFromEnvironment, - Dial: (&net.Dialer{ - Timeout: 30 * time.Second, - }).Dial, - TLSHandshakeTimeout: 5 * time.Second, -} -var netClient WebhookClient = &http.Client{ - Timeout: time.Second * 30, - Transport: netTransport, -} - func (ns *NotificationService) sendWebRequestSync(ctx context.Context, webhook *Webhook) error { if webhook.HttpMethod == "" { webhook.HttpMethod = http.MethodPost @@ -85,7 +71,7 @@ func (ns *NotificationService) sendWebRequestSync(ctx context.Context, webhook * request.Header.Set(k, v) } - resp, err := netClient.Do(request) + resp, err := alertingReceivers.NewTLSClient(webhook.TLSConfig).Do(request) if err != nil { return redactURL(err) }