Prometheus: Add error source, don't swallow errors (#96511)

add error source, don't swallow errors
This commit is contained in:
Brendan O'Handley 2024-11-25 10:30:13 -06:00 committed by GitHub
parent df628951f5
commit 287ce2ef40
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -234,10 +234,7 @@ func (s *QueryData) fetch(traceCtx context.Context, client *client.Client, q *mo
func (s *QueryData) rangeQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse { func (s *QueryData) rangeQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse {
res, err := c.QueryRange(ctx, q) res, err := c.QueryRange(ctx, q)
if err != nil { if err != nil {
return backend.DataResponse{ return addErrorSourceToDataResponse(err)
Error: err,
Status: backend.StatusBadGateway,
}
} }
defer func() { defer func() {
@ -253,16 +250,14 @@ func (s *QueryData) rangeQuery(ctx context.Context, c *client.Client, q *models.
func (s *QueryData) instantQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse { func (s *QueryData) instantQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse {
res, err := c.QueryInstant(ctx, q) res, err := c.QueryInstant(ctx, q)
if err != nil { if err != nil {
return backend.DataResponse{ return addErrorSourceToDataResponse(err)
Error: err,
Status: backend.StatusBadGateway,
}
} }
// This is only for health check fall back scenario // This is only for health check fall back scenario
if res.StatusCode != 200 && q.RefId == "__healthcheck__" { if res.StatusCode != 200 && q.RefId == "__healthcheck__" {
return backend.DataResponse{ return backend.DataResponse{
Error: errors.New(res.Status), Error: errors.New(res.Status),
ErrorSource: backend.ErrorSourceFromHTTPStatus(res.StatusCode),
} }
} }
@ -279,9 +274,14 @@ func (s *QueryData) instantQuery(ctx context.Context, c *client.Client, q *model
func (s *QueryData) exemplarQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse { func (s *QueryData) exemplarQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse {
res, err := c.QueryExemplars(ctx, q) res, err := c.QueryExemplars(ctx, q)
if err != nil { if err != nil {
return backend.DataResponse{ response := backend.DataResponse{
Error: err, Error: err,
} }
if backend.IsDownstreamHTTPError(err) {
response.ErrorSource = backend.ErrorSourceDownstream
}
return response
} }
defer func() { defer func() {
@ -304,3 +304,15 @@ func addDataResponse(res *backend.DataResponse, dr *backend.DataResponse) {
} }
dr.Frames = append(dr.Frames, res.Frames...) dr.Frames = append(dr.Frames, res.Frames...)
} }
func addErrorSourceToDataResponse(err error) backend.DataResponse {
response := backend.DataResponse{
Error: err,
Status: backend.StatusBadGateway,
}
if backend.IsDownstreamHTTPError(err) {
response.ErrorSource = backend.ErrorSourceDownstream
}
return response
}