Search: add request metrics (#53028)

* Search: add request metrics

* Search: consistent naming

* Search: remove success counter

* Search: update buckets

* Search: remove log
This commit is contained in:
Artur Wierzbicki 2022-08-01 15:23:41 +04:00 committed by GitHub
parent cea4b3fb19
commit 784cfcf2b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
@ -13,10 +14,40 @@ import (
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/services/store"
"github.com/grafana/grafana/pkg/setting"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/grafana/grafana-plugin-sdk-go/backend"
)
var (
namespace = "grafana"
subsystem = "search"
dashboardSearchFailureRequestsCounter = promauto.NewCounterVec(
prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "dashboard_search_failures_total",
Help: "A counter for failed dashboard search requests",
},
[]string{"reason"},
)
dashboardSearchSuccessRequestsDuration = promauto.NewHistogram(
prometheus.HistogramOpts{
Name: "dashboard_search_successes_duration_seconds",
Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25, 50, 100},
Namespace: namespace,
Subsystem: subsystem,
})
dashboardSearchFailureRequestsDuration = promauto.NewHistogram(
prometheus.HistogramOpts{
Name: "dashboard_search_failures_duration_seconds",
Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25, 50, 100},
Namespace: namespace,
Subsystem: subsystem,
})
)
type StandardSearchService struct {
registry.BackgroundService
@ -150,27 +181,53 @@ func (s *StandardSearchService) getUser(ctx context.Context, backendUser *backen
}
func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *backend.User, orgID int64, q DashboardQuery) *backend.DataResponse {
start := time.Now()
query := s.doDashboardQuery(ctx, user, orgID, q)
duration := time.Since(start).Seconds()
if query.Error != nil {
dashboardSearchFailureRequestsDuration.Observe(duration)
} else {
dashboardSearchSuccessRequestsDuration.Observe(duration)
}
return query
}
func (s *StandardSearchService) doDashboardQuery(ctx context.Context, user *backend.User, orgID int64, q DashboardQuery) *backend.DataResponse {
rsp := &backend.DataResponse{}
signedInUser, err := s.getUser(ctx, user, orgID)
if err != nil {
dashboardSearchFailureRequestsCounter.With(prometheus.Labels{
"reason": "get_user_error",
}).Inc()
rsp.Error = err
return rsp
}
filter, err := s.auth.GetDashboardReadFilter(signedInUser)
if err != nil {
dashboardSearchFailureRequestsCounter.With(prometheus.Labels{
"reason": "get_dashboard_filter_error",
}).Inc()
rsp.Error = err
return rsp
}
index, err := s.dashboardIndex.getOrCreateOrgIndex(ctx, orgID)
if err != nil {
dashboardSearchFailureRequestsCounter.With(prometheus.Labels{
"reason": "get_index_error",
}).Inc()
rsp.Error = err
return rsp
}
err = s.dashboardIndex.sync(ctx)
if err != nil {
dashboardSearchFailureRequestsCounter.With(prometheus.Labels{
"reason": "dashboard_index_sync_error",
}).Inc()
rsp.Error = err
return rsp
}
@ -183,5 +240,11 @@ func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *back
}
}
if response.Error != nil {
dashboardSearchFailureRequestsCounter.With(prometheus.Labels{
"reason": "search_query_error",
}).Inc()
}
return response
}