Instrumentation: Handle context.Canceled (#75867)

Ref #68480

Co-authored-by: Giuseppe Guerra <giuseppe.guerra@grafana.com>

contribute/backend/errors.md

@@ -31,6 +31,7 @@ functions, e.g.
 - `errutil.Forbidden(messageID, opts...)`
 - `errutil.TooManyRequests(messageID, opts...)`
 - `errutil.NotImplemented(messageID, opts...)`
+- `errutil.ClientClosedRequest(messageID, opts...)`
 
 Above functions uses `errutil.NewBase(status, messageID, opts...)` under the covers, and that function should in general only be used outside the `errutil` package for `errutil.StatusUnknown`, e.g. when there are no accurate status code available/provided.
 

devenv/datasources.yaml

@@ -74,8 +74,14 @@ datasources:
     type: prometheus
     access: proxy
     url: http://localhost:3011
+    basicAuth: true #username: admin, password: admin
+    basicAuthUser: admin
     jsonData:
       manageAlerts: false
+      prometheusType: Prometheus #Cortex | Mimir | Prometheus | Thanos
+      prometheusVersion: 2.40.0
+    secureJsonData:
+      basicAuthPassword: admin #https://grafana.com/docs/grafana/latest/administration/provisioning/#using-environment-variables
 
   - name: gdev-testdata
     isDefault: true

pkg/api/response/response.go

@@ -2,6 +2,7 @@ package response
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -18,6 +19,9 @@ import (
 	"github.com/grafana/grafana/pkg/util/errutil"
 )
 
+var errRequestCanceledBase = errutil.ClientClosedRequest("api.requestCanceled",
+	errutil.WithPublicMessage("Request canceled"))
+
 // Response is an HTTP response interface.
 type Response interface {
 	// WriteTo writes to a context.
@@ -286,12 +290,18 @@ func Err(err error) *NormalResponse {
 // The signature is equivalent to that of Error which allows us to
 // rename this to Error when we're confident that that would be safe to
 // do.
+// If the error provided is not an errutil.Error and is/wraps context.Canceled
+// the function returns an Err(errRequestCanceledBase).
 func ErrOrFallback(status int, message string, err error) *NormalResponse {
 	grafanaErr := errutil.Error{}
 	if errors.As(err, &grafanaErr) {
 		return Err(err)
 	}
 
+	if errors.Is(err, context.Canceled) {
+		return Err(errRequestCanceledBase.Errorf("response: request canceled: %w", err))
+	}
+
 	return Error(status, message, err)
 }
 

pkg/plugins/errors.go

@@ -29,4 +29,10 @@ var (
 	ErrPluginDownstreamErrorBase = errutil.Internal("plugin.downstreamError",
 		errutil.WithPublicMessage("An error occurred within the plugin"),
 		errutil.WithDownstream())
+
+	// ErrPluginRequestCanceledErrorBase error returned when a plugin request
+	// is cancelled by the client (context is cancelled).
+	// Exposed as a base error to wrap it with plugin cancelled errors.
+	ErrPluginRequestCanceledErrorBase = errutil.ClientClosedRequest("plugin.requestCanceled",
+		errutil.WithPublicMessage("Plugin request canceled"))
 )

pkg/plugins/manager/client/client.go

@@ -59,6 +59,10 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
 			return nil, err
 		}
 
+		if errors.Is(err, context.Canceled) {
+			return nil, plugins.ErrPluginRequestCanceledErrorBase.Errorf("client: query data request canceled: %w", err)
+		}
+
 		return nil, plugins.ErrPluginDownstreamErrorBase.Errorf("client: failed to query data: %w", err)
 	}
 
@@ -111,6 +115,10 @@ func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceReq
 
 	err := p.CallResource(ctx, req, wrappedSender)
 	if err != nil {
+		if errors.Is(err, context.Canceled) {
+			return plugins.ErrPluginRequestCanceledErrorBase.Errorf("client: call resource request canceled: %w", err)
+		}
+
 		return plugins.ErrPluginDownstreamErrorBase.Errorf("client: failed to call resources: %w", err)
 	}
 
@@ -129,6 +137,10 @@ func (s *Service) CollectMetrics(ctx context.Context, req *backend.CollectMetric
 
 	resp, err := p.CollectMetrics(ctx, req)
 	if err != nil {
+		if errors.Is(err, context.Canceled) {
+			return nil, plugins.ErrPluginRequestCanceledErrorBase.Errorf("client: collect metrics request canceled: %w", err)
+		}
+
 		return nil, plugins.ErrPluginDownstreamErrorBase.Errorf("client: failed to collect metrics: %w", err)
 	}
 
@@ -155,6 +167,10 @@ func (s *Service) CheckHealth(ctx context.Context, req *backend.CheckHealthReque
 			return nil, err
 		}
 
+		if errors.Is(err, context.Canceled) {
+			return nil, plugins.ErrPluginRequestCanceledErrorBase.Errorf("client: check health request canceled: %w", err)
+		}
+
 		return nil, plugins.ErrPluginHealthCheck.Errorf("client: failed to check health: %w", err)
 	}
 

pkg/plugins/manager/client/client_test.go

@@ -41,6 +41,10 @@ func TestQueryData(t *testing.T) {
 				err:           errors.New("surprise surprise"),
 				expectedError: plugins.ErrPluginDownstreamErrorBase,
 			},
+			{
+				err:           context.Canceled,
+				expectedError: plugins.ErrPluginRequestCanceledErrorBase,
+			},
 		}
 
 		for _, tc := range tcs {
@@ -99,6 +103,10 @@ func TestCheckHealth(t *testing.T) {
 				err:           errors.New("surprise surprise"),
 				expectedError: plugins.ErrPluginHealthCheck,
 			},
+			{
+				err:           context.Canceled,
+				expectedError: plugins.ErrPluginRequestCanceledErrorBase,
+			},
 		}
 
 		for _, tc := range tcs {

pkg/util/errutil/errors.go

@@ -133,6 +133,17 @@ func TooManyRequests(msgID string, opts ...BaseOpt) Base {
 	return NewBase(StatusTooManyRequests, msgID, opts...)
 }
 
+// ClientClosedRequest initializes a new [Base] error with reason StatusClientClosedRequest
+// that is used to construct [Error]. The msgID is passed to the caller
+// to serve as the base for user facing error messages.
+//
+// msgID should be structured as component.errorBrief, for example
+//
+//	plugin.requestCanceled
+func ClientClosedRequest(msgID string, opts ...BaseOpt) Base {
+	return NewBase(StatusClientClosedRequest, msgID, opts...)
+}
+
 // NotImplemented initializes a new [Base] error with reason StatusNotImplemented
 // that is used to construct [Error]. The msgID is passed to the caller
 // to serve as the base for user facing error messages.

pkg/util/errutil/status.go

@@ -28,6 +28,13 @@ const (
 	// parameters or payload for the request.
 	// HTTP status code 400.
 	StatusBadRequest CoreStatus = "Bad request"
+	// StatusClientClosedRequest means that a client closes the connection
+	// while the server is processing the request.
+	//
+	// This is a non-standard HTTP status code introduced by nginx, see
+	// https://httpstatus.in/499/ for more information.
+	// HTTP status code 499.
+	StatusClientClosedRequest CoreStatus = "Client closed request"
 	// StatusValidationFailed means that the server was able to parse
 	// the payload for the request but it failed one or more validation
 	// checks.
@@ -57,6 +64,11 @@ const (
 	StatusGatewayTimeout CoreStatus = "Gateway timeout"
 )
 
+// HTTPStatusClientClosedRequest A non-standard status code introduced by nginx
+// for the case when a client closes the connection while nginx is processing
+// the request. See https://httpstatus.in/499/ for more information.
+const HTTPStatusClientClosedRequest = 499
+
 // StatusReason allows for wrapping of CoreStatus.
 type StatusReason interface {
 	Status() CoreStatus
@@ -84,6 +96,8 @@ func (s CoreStatus) HTTPStatus() int {
 		return http.StatusTooManyRequests
 	case StatusBadRequest, StatusValidationFailed:
 		return http.StatusBadRequest
+	case StatusClientClosedRequest:
+		return HTTPStatusClientClosedRequest
 	case StatusNotImplemented:
 		return http.StatusNotImplemented
 	case StatusBadGateway: