IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Prometheus: Check both error and http status for source of error (#96060)

Browse Source 
* check error for source and check http status for source

* refactor

* add source to health check status code as well

* remove error where it will be nil
This commit is contained in:
Brendan O'Handley 2024-11-08 15:19:14 -06:00 committed by GitHub
parent 5a143be653
commit 3f81f0d145
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 37 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
pkg/promlib/querydata/request.go
View File

@ -234,9 +234,13 @@ func (s *QueryData) fetch(traceCtx context.Context, client *client.Client, q *mo
func (s *QueryData) rangeQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse { func (s *QueryData) rangeQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse {
res, err := c.QueryRange(ctx, q) res, err := c.QueryRange(ctx, q)
if err != nil { if err != nil {
return addErrorSourceToDataResponse(err)
}
if res.StatusCode/100 != 2 {
// for differentiating the source of http errors by status code
return backend.DataResponse{ return backend.DataResponse{
Error: err, ErrorSource: backend.ErrorSourceFromHTTPStatus(res.StatusCode),
Status: backend.StatusBadGateway,
} }
} }
@ -253,16 +257,22 @@ func (s *QueryData) rangeQuery(ctx context.Context, c *client.Client, q *models.
func (s *QueryData) instantQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse { func (s *QueryData) instantQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse {
res, err := c.QueryInstant(ctx, q) res, err := c.QueryInstant(ctx, q)
if err != nil { if err != nil {
return backend.DataResponse{ // confirm that it is a downstream error
Error: err, return addErrorSourceToDataResponse(err)
Status: backend.StatusBadGateway,
}
} }
// This is only for health check fall back scenario if res.StatusCode/100 != 2 {
if res.StatusCode != 200 && q.RefId == "__healthcheck__" { // This is only for health check fall back scenario
// add more details to the health check error
if q.RefId == "__healthcheck__" {
return backend.DataResponse{
Error: errors.New(res.Status),
ErrorSource: backend.ErrorSourceFromHTTPStatus(res.StatusCode),
}
}
// for differentiating the source of http errors by status code
return backend.DataResponse{ return backend.DataResponse{
Error: errors.New(res.Status), ErrorSource: backend.ErrorSourceFromHTTPStatus(res.StatusCode),
} }
} }
@ -279,8 +289,13 @@ func (s *QueryData) instantQuery(ctx context.Context, c *client.Client, q *model
func (s *QueryData) exemplarQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse { func (s *QueryData) exemplarQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse {
res, err := c.QueryExemplars(ctx, q) res, err := c.QueryExemplars(ctx, q)
if err != nil { if err != nil {
return addErrorSourceToDataResponse(err)
}
if res.StatusCode/100 != 2 {
// for differentiating the source of http errors by status code
return backend.DataResponse{ return backend.DataResponse{
Error: err, ErrorSource: backend.ErrorSourceFromHTTPStatus(res.StatusCode),
} }
} }
@ -304,3 +319,15 @@ func addDataResponse(res *backend.DataResponse, dr *backend.DataResponse) {
} }
dr.Frames = append(dr.Frames, res.Frames...) dr.Frames = append(dr.Frames, res.Frames...)
} }
func addErrorSourceToDataResponse(err error) backend.DataResponse {
response := backend.DataResponse{
Error: err,
Status: backend.StatusBadGateway,
}
if backend.IsDownstreamHTTPError(err) {
response.ErrorSource = backend.ErrorSourceDownstream
}
return response
}