Alerting: Support tls config for webhook receiver (#93513)

Adds the ability to configure tls settings on the webhook receiver (e.g. to skip server certificate validation)

.github/CODEOWNERS

@@ -194,6 +194,7 @@
 /devenv/dev-dashboards-without-uid/ @grafana/dashboards-squad
 /devenv/dev-dashboards/ @grafana/dashboards-squad
 /devenv/docker/blocks/alert_webhook_listener/ @grafana/alerting-backend
+/devenv/docker/blocks/caddy_tls/ @grafana/alerting-backend
 /devenv/docker/blocks/clickhouse/ @grafana/partner-datasources
 /devenv/docker/blocks/collectd/ @grafana/observability-metrics
 /devenv/docker/blocks/etcd @grafana/grafana-app-platform-squad

devenv/docker/blocks/caddy_tls/README.md

@@ -0,0 +1,33 @@
+# TLS Caddy Server
+
+Starts a [Caddy server](https://caddyserver.com/) with TLS configured.
+
+## Setup
+
+- Caddy is setup to run on port 2081, so when configuring the webhook receiver in Grafana Alerting you should use the
+following the following URL: `https://localhost:2081`
+- Also, Caddy is configured to use a self-signed certificate and to check the client certificate (`require_and_verify` mode)
+- Caddy is setup to log requests and has debug mode enabled to make it easier to investigate possible issues
+
+## TLS Certificates
+
+If you want to configure a webhook contact point in Grafana Alerting with TLS, you need to provide a certificate and key.
+
+You can find them in `/etc/caddy` directory in the container:
+
+``` shell
+docker exec devenv-caddy_tls-1 ls /etc/caddy/
+```
+
+### CA Certificate
+
+``` shell
+docker exec devenv-caddy_tls-1 cat /etc/caddy/ca.pem
+```
+
+### Client certificates
+
+``` shell
+docker exec devenv-caddy_tls-1 cat /etc/caddy/client.pem
+docker exec devenv-caddy_tls-1 cat /etc/caddy/client.key
+```

devenv/docker/blocks/caddy_tls/build/Caddyfile

@@ -0,0 +1,14 @@
+{
+    debug
+}
+
+localhost:2081 {
+    log
+    tls /etc/caddy/server.pem /etc/caddy/server.key {
+      ca_root /etc/caddy/ca.pem
+      client_auth {
+        mode require_and_verify
+        trust_pool file /etc/caddy/client.pem /etc/caddy/ca.pem
+      }
+    }
+}

devenv/docker/blocks/caddy_tls/build/Dockerfile

@@ -0,0 +1,12 @@
+FROM caddy:2.8.4-alpine
+
+WORKDIR /etc/caddy
+EXPOSE 2081
+
+COPY Caddyfile ./Caddyfile
+COPY san.cnf ./san.cnf
+COPY gen_certs.sh ./gen_certs.sh
+
+RUN apk update && apk upgrade --no-cache && apk add openssl
+
+RUN ./gen_certs.sh

devenv/docker/blocks/caddy_tls/build/gen_certs.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+DAYS_VALID=3650
+
+# Create CA certificate
+openssl genpkey -algorithm RSA -out ca.key
+openssl req -new -x509 -days $DAYS_VALID -key ca.key -out ca.pem -subj "/CN=My CA"
+
+# Create server certificate
+openssl genpkey -algorithm RSA -out server.key
+openssl req -new -key server.key -out server.csr -subj "/CN=localhost"
+openssl x509 -req -days $DAYS_VALID -in server.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out server.pem -extfile san.cnf -extensions v3_req
+
+# Create client key and certificate
+openssl genpkey -algorithm RSA -out client.key
+openssl req -new -key client.key -out client.csr -subj "/CN=Client"
+openssl x509 -req -days $DAYS_VALID -in client.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out client.pem -extfile san.cnf -extensions v3_req

devenv/docker/blocks/caddy_tls/build/san.cnf

@@ -0,0 +1,7 @@
+[ v3_req ]
+subjectAltName = @alt_names
+
+[ alt_names ]
+DNS.1 = localhost
+IP.1 = 127.0.0.1
+IP.2 = ::1

devenv/docker/blocks/caddy_tls/docker-compose.yaml

@@ -0,0 +1,5 @@
+  caddy_tls:
+    build:
+      context: docker/blocks/caddy_tls/build
+    ports:
+      - "2081:2081"

docs/sources/administration/provisioning/index.md

@@ -615,11 +615,21 @@ The following sections detail the supported settings and secure settings for eac
 #### Alert notification `webhook`
 
 | Name        | Secure setting |
-| ---------- | -------------- |
+| ----------- | -------------- |
 | url         |                |
-| httpMethod |                |
+| http_method |                |
 | username    |                |
 | password    | yes            |
+| tls_config  |                |
+
+##### TLS config
+
+| Name               | Secure setting |
+| ------------------ | -------------- |
+| insecureSkipVerify |                |
+| clientCertificate  | yes            |
+| clientKey          | yes            |
+| caCertificate      | yes            |
 
 #### Alert notification `googlechat`
 

docs/sources/alerting/set-up/provision-alerting-resources/file-provisioning/index.md

@@ -616,6 +616,16 @@ settings:
   authorization_credentials: abc123
   # <string>
   maxAlerts: '10'
+  # <map>
+  tlsConfig:
+    # <bool>
+    insecureSkipVerify: false
+    # <string>
+    clientCertificate: certificate in PEM format
+    # <string>
+    clientKey: key in PEM format
+    # <string>
+    caCertificate: CA certificate in PEM format
 ```
 
 {{< /collapse >}}

go.mod

@@ -72,7 +72,7 @@ require (
 	github.com/googleapis/gax-go/v2 v2.13.0 // @grafana/grafana-backend-group
 	github.com/gorilla/mux v1.8.1 // @grafana/grafana-backend-group
 	github.com/gorilla/websocket v1.5.0 // @grafana/grafana-app-platform-squad
-	github.com/grafana/alerting v0.0.0-20241010165806-807ddf183724 // @grafana/alerting-backend
+	github.com/grafana/alerting v0.0.0-20241021123319-be61d61f71e7 // @grafana/alerting-backend
 	github.com/grafana/authlib v0.0.0-20241018103850-afc1195d8240 // @grafana/identity-access-team
 	github.com/grafana/authlib/claims v0.0.0-20241018085709-130ad686d80e // @grafana/identity-access-team
 	github.com/grafana/codejen v0.0.3 // @grafana/dataviz-squad

go.sum

@@ -2243,8 +2243,8 @@ github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grafana/alerting v0.0.0-20241010165806-807ddf183724 h1:u+ZM5TLkdeEoSWXgYWxc4XRfPHhXpR63MyHXJxbBLrc=
+github.com/grafana/alerting v0.0.0-20241021123319-be61d61f71e7 h1:lsM/QscEX+ZDIJm48ynQscH+msETyGYV6ug8L4f2DtM=
-github.com/grafana/alerting v0.0.0-20241010165806-807ddf183724/go.mod h1:QsnoKX/iYZxA4Cv+H+wC7uxutBD8qi8ZW5UJvD2TYmU=
+github.com/grafana/alerting v0.0.0-20241021123319-be61d61f71e7/go.mod h1:QsnoKX/iYZxA4Cv+H+wC7uxutBD8qi8ZW5UJvD2TYmU=
 github.com/grafana/authlib v0.0.0-20241018103850-afc1195d8240 h1:bBn6sCbBjxjYlvs5JAIGHQSOs8xbDEBWbezxarA/DDo=
 github.com/grafana/authlib v0.0.0-20241018103850-afc1195d8240/go.mod h1:RKqhn8E5PY2k5Xo6X8FHFgP45/qt9qqfAY7YYJ2mtB8=
 github.com/grafana/authlib/claims v0.0.0-20241018085709-130ad686d80e h1:I0sSXcqdt/ttiOJ/BVhpfa2q/xAyWSweQwaypGmvLss=

pkg/services/ngalert/api/tooling/definitions/contact_points.go

@@ -297,6 +297,7 @@ type WebhookIntegration struct {
 	Password                 *Secret    `json:"password,omitempty" yaml:"password,omitempty" hcl:"basic_auth_password"`
 	Title                    *string    `json:"title,omitempty" yaml:"title,omitempty" hcl:"title"`
 	Message                  *string    `json:"message,omitempty" yaml:"message,omitempty" hcl:"message"`
+	TLSConfig                *TLSConfig `json:"tlsConfig,omitempty" yaml:"tlsConfig,omitempty" hcl:"tlsConfig,block"`
 }
 
 type WecomIntegration struct {

pkg/services/ngalert/notifier/channels_config/available_channels.go

@@ -964,6 +964,48 @@ func GetAvailableNotifiers() []*NotifierPlugin {
 					PropertyName: "message",
 					Placeholder:  alertingTemplates.DefaultMessageEmbed,
 				},
+				{
+					Label:        "TLS",
+					PropertyName: "tlsConfig",
+					Description:  "TLS configuration options",
+					Element:      ElementTypeSubform,
+					SubformOptions: []NotifierOption{
+						{
+							Label:        "Disable certificate verification",
+							Element:      ElementTypeCheckbox,
+							Description:  "Do not verify the server's certificate chain and host name.",
+							PropertyName: "insecureSkipVerify",
+							Required:     false,
+						},
+						{
+							Label:        "CA Certificate",
+							Element:      ElementTypeTextArea,
+							Description:  "Certificate in PEM format to use when verifying the server's certificate chain.",
+							InputType:    InputTypeText,
+							PropertyName: "caCertificate",
+							Required:     false,
+							Secure:       true,
+						},
+						{
+							Label:        "Client Certificate",
+							Element:      ElementTypeTextArea,
+							Description:  "Client certificate in PEM format to use when connecting to the server.",
+							InputType:    InputTypeText,
+							PropertyName: "clientCertificate",
+							Required:     false,
+							Secure:       true,
+						},
+						{
+							Label:        "Client Key",
+							Element:      ElementTypeTextArea,
+							Description:  "Client key in PEM format to use when connecting to the server.",
+							InputType:    InputTypeText,
+							PropertyName: "clientKey",
+							Required:     false,
+							Secure:       true,
+						},
+					},
+				},
 			},
 		},
 		{

pkg/services/ngalert/notifier/channels_config/available_channels_test.go

@@ -22,7 +22,7 @@ func TestGetSecretKeysForContactPointType(t *testing.T) {
 		{receiverType: "sensugo", expectedSecretFields: []string{"apikey"}},
 		{receiverType: "teams", expectedSecretFields: []string{}},
 		{receiverType: "telegram", expectedSecretFields: []string{"bottoken"}},
-		{receiverType: "webhook", expectedSecretFields: []string{"password", "authorization_credentials"}},
+		{receiverType: "webhook", expectedSecretFields: []string{"password", "authorization_credentials", "tlsConfig.caCertificate", "tlsConfig.clientCertificate", "tlsConfig.clientKey"}},
 		{receiverType: "wecom", expectedSecretFields: []string{"url", "secret"}},
 		{receiverType: "prometheus-alertmanager", expectedSecretFields: []string{"basicAuthPassword"}},
 		{receiverType: "discord", expectedSecretFields: []string{"url"}},

pkg/services/ngalert/notifier/sender.go

@@ -22,6 +22,7 @@ func (s sender) SendWebhook(ctx context.Context, cmd *receivers.SendWebhookSetti
 		HttpHeader:  cmd.HTTPHeader,
 		ContentType: cmd.ContentType,
 		Validation:  cmd.Validation,
+		TLSConfig:   cmd.TLSConfig,
 	})
 }
 

pkg/services/notifications/models.go

@@ -1,6 +1,7 @@
 package notifications
 
 import (
+	"crypto/tls"
 	"errors"
 
 	"github.com/grafana/grafana/pkg/services/user"
@@ -42,6 +43,7 @@ type SendWebhookSync struct {
 	HttpHeader  map[string]string
 	ContentType string
 	Validation  func(body []byte, statusCode int) error
+	TLSConfig   *tls.Config
 }
 
 type SendResetPasswordEmailCommand struct {

pkg/services/notifications/notifications.go

@@ -120,7 +120,6 @@ func (ns *NotificationService) Run(ctx context.Context) error {
 		select {
 		case webhook := <-ns.webhookQueue:
 			err := ns.sendWebRequestSync(context.Background(), webhook)
-
 			if err != nil {
 				ns.log.Error("Failed to send webrequest ", "error", err)
 			}
@@ -155,6 +154,7 @@ func (ns *NotificationService) SendWebhookSync(ctx context.Context, cmd *SendWeb
 		HttpMethod:  cmd.HttpMethod,
 		HttpHeader:  cmd.HttpHeader,
 		ContentType: cmd.ContentType,
+		TLSConfig:   cmd.TLSConfig,
 		Validation:  cmd.Validation,
 	})
 }
@@ -211,7 +211,6 @@ func (ns *NotificationService) SendEmailCommandHandlerSync(ctx context.Context,
 		Subject:       cmd.Subject,
 		ReplyTo:       cmd.ReplyTo,
 	})
-
 	if err != nil {
 		return err
 	}
@@ -222,7 +221,6 @@ func (ns *NotificationService) SendEmailCommandHandlerSync(ctx context.Context,
 
 func (ns *NotificationService) SendEmailCommandHandler(ctx context.Context, cmd *SendEmailCommand) error {
 	message, err := ns.buildEmailMessage(cmd)
-
 	if err != nil {
 		return err
 	}
@@ -304,7 +302,6 @@ func (ns *NotificationService) signUpStartedHandler(ctx context.Context, evt *ev
 			"SignUpUrl": setting.ToAbsUrl(fmt.Sprintf("signup/?email=%s&code=%s", url.QueryEscape(evt.Email), url.QueryEscape(evt.Code))),
 		},
 	})
-
 	if err != nil {
 		return err
 	}

pkg/services/notifications/testing.go

@@ -33,11 +33,3 @@ func NewFakeDisconnectedMailer() *FakeDisconnectedMailer {
 func (fdm *FakeDisconnectedMailer) Send(ctx context.Context, messages ...*Message) (int, error) {
 	return 0, fmt.Errorf("connect: connection refused")
 }
-
-// NetClient is used to export original in test.
-var NetClient = &netClient
-
-// SetWebhookClient is used to mock in test.
-func SetWebhookClient(client WebhookClient) {
-	netClient = client
-}

pkg/services/notifications/webhook.go

@@ -7,10 +7,10 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/url"
-	"time"
+
+	alertingReceivers "github.com/grafana/alerting/receivers"
 
 	"github.com/grafana/grafana/pkg/util"
 )
@@ -23,6 +23,7 @@ type Webhook struct {
 	HttpMethod  string
 	HttpHeader  map[string]string
 	ContentType string
+	TLSConfig   *tls.Config
 
 	// Validation is a function that will validate the response body and statusCode of the webhook. Any returned error will cause the webhook request to be considered failed.
 	// This can be useful when a webhook service communicates failures in creative ways, such as using the response body instead of the status code.
@@ -34,21 +35,6 @@ type WebhookClient interface {
 	Do(req *http.Request) (*http.Response, error)
 }
 
-var netTransport = &http.Transport{
-	TLSClientConfig: &tls.Config{
-		Renegotiation: tls.RenegotiateFreelyAsClient,
-	},
-	Proxy: http.ProxyFromEnvironment,
-	Dial: (&net.Dialer{
-		Timeout: 30 * time.Second,
-	}).Dial,
-	TLSHandshakeTimeout: 5 * time.Second,
-}
-var netClient WebhookClient = &http.Client{
-	Timeout:   time.Second * 30,
-	Transport: netTransport,
-}
-
 func (ns *NotificationService) sendWebRequestSync(ctx context.Context, webhook *Webhook) error {
 	if webhook.HttpMethod == "" {
 		webhook.HttpMethod = http.MethodPost
@@ -85,7 +71,7 @@ func (ns *NotificationService) sendWebRequestSync(ctx context.Context, webhook *
 		request.Header.Set(k, v)
 	}
 
-	resp, err := netClient.Do(request)
+	resp, err := alertingReceivers.NewTLSClient(webhook.TLSConfig).Do(request)
 	if err != nil {
 		return redactURL(err)
 	}