IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Prometheus: Add interpolation for built-in-time variables to backend (#39051)

Browse Source 
* Add grafana variable interpolation to backend

* Remove newlines
This commit is contained in:
Ivana Huckova 2021-09-10 16:56:15 -04:00 committed by GitHub
parent a30f560b35
commit 4952f6fc58
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 280 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
pkg/tsdb/intervalv2/intervalv2.go
View File

@ -82,31 +82,40 @@ func (ic *intervalCalculator) CalculateSafeInterval(timerange backend.TimeRange,
// queryIntervalMS is a pre-calculated numeric representation of the query interval in milliseconds.
func GetIntervalFrom(dsInterval, queryInterval string, queryIntervalMS int64, defaultInterval time.Duration) (time.Duration, error) {
// Apparently we are setting default value of queryInterval to 0s now
if queryInterval == "0s" {
queryInterval = ""
interval := queryInterval
if interval == "0s" {
interval = ""
}
if queryInterval == "" {
if interval == "" {
if queryIntervalMS != 0 {
return time.Duration(queryIntervalMS) * time.Millisecond, nil
}
}
interval := queryInterval
if queryInterval == "" && dsInterval != "" {
if interval == "" && dsInterval != "" {
interval = dsInterval
}
if interval == "" {
return defaultInterval, nil
}
interval = strings.Replace(strings.Replace(interval, "<", "", 1), ">", "", 1)
isPureNum, err := regexp.MatchString(`^\d+$`, interval)
parsedInterval, err := ParseIntervalStringToTimeDuration(interval)
if err != nil {
return time.Duration(0), err
}
return parsedInterval, nil
}
func ParseIntervalStringToTimeDuration(interval string) (time.Duration, error) {
formattedInterval := strings.Replace(strings.Replace(interval, "<", "", 1), ">", "", 1)
isPureNum, err := regexp.MatchString(`^\d+$`, formattedInterval)
if err != nil {
return time.Duration(0), err
}
if isPureNum {
interval += "s"
formattedInterval += "s"
}
parsedInterval, err := time.ParseDuration(interval)
parsedInterval, err := time.ParseDuration(formattedInterval)
if err != nil {
return time.Duration(0), err
}

90
pkg/tsdb/prometheus/prometheus.go
View File

@ -5,8 +5,10 @@ import (
"encoding/json"
"errors"
"fmt"
"math"
"net/http"
"regexp"
"strconv"
"strings"
"time"
@ -15,7 +17,6 @@ import (
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
@ -41,6 +42,17 @@ type DatasourceInfo struct {
TimeInterval string
}
type QueryModel struct {
Expr string `json:"expr"`
LegendFormat string `json:"legendFormat"`
Interval string `json:"interval"`
IntervalMS int64 `json:"intervalMS"`
StepMode string `json:"stepMode"`
RangeQuery bool `json:"range"`
InstantQuery bool `json:"instant"`
IntervalFactor int64 `json:"intervalFactor"`
}
type Service struct {
httpClientProvider httpclient.Provider
intervalCalculator intervalv2.Calculator
@ -130,7 +142,7 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
Responses: backend.Responses{},
}
queries, err := s.parseQuery(req.Queries, dsInfo)
queries, err := s.parseQuery(req, dsInfo)
if err != nil {
return &result, err
}
@ -225,48 +237,59 @@ func formatLegend(metric model.Metric, query *PrometheusQuery) string {
return string(result)
}
func (s *Service) parseQuery(queries []backend.DataQuery, dsInfo *DatasourceInfo) ([]*PrometheusQuery, error) {
func (s *Service) parseQuery(queryContext *backend.QueryDataRequest, dsInfo *DatasourceInfo) ([]*PrometheusQuery, error) {
qs := []*PrometheusQuery{}
for _, queryModel := range queries {
jsonModel, err := simplejson.NewJson(queryModel.JSON)
if err != nil {
return nil, err
}
expr, err := jsonModel.Get("expr").String()
for _, query := range queryContext.Queries {
model := &QueryModel{}
err := json.Unmarshal(query.JSON, model)
if err != nil {
return nil, err
}
format := jsonModel.Get("legendFormat").MustString("")
start := queryModel.TimeRange.From
end := queryModel.TimeRange.To
queryInterval := jsonModel.Get("interval").MustString("")
minInterval, err := intervalv2.GetIntervalFrom(dsInfo.TimeInterval, queryInterval, 0, 15*time.Second)
//Calculate interval
queryInterval := model.Interval
//If we are using variable or interval/step, we will replace it with calculated interval
if queryInterval == "$__interval" || queryInterval == "$__interval_ms" {
queryInterval = ""
}
minInterval, err := intervalv2.GetIntervalFrom(dsInfo.TimeInterval, queryInterval, model.IntervalMS, 15*time.Second)
if err != nil {
return nil, err
}
calculatedInterval := s.intervalCalculator.Calculate(queries[0].TimeRange, minInterval)
safeInterval := s.intervalCalculator.CalculateSafeInterval(queries[0].TimeRange, int64(safeRes))
calculatedInterval := s.intervalCalculator.Calculate(query.TimeRange, minInterval)
safeInterval := s.intervalCalculator.CalculateSafeInterval(query.TimeRange, int64(safeRes))
adjustedInterval := safeInterval.Value
if calculatedInterval.Value > safeInterval.Value {
adjustedInterval = calculatedInterval.Value
}
intervalFactor := jsonModel.Get("intervalFactor").MustInt64(1)
step := time.Duration(int64(adjustedInterval) * intervalFactor)
intervalFactor := model.IntervalFactor
if intervalFactor == 0 {
intervalFactor = 1
}
interval := time.Duration(int64(adjustedInterval) * intervalFactor)
intervalMs := int64(interval / time.Millisecond)
rangeS := query.TimeRange.To.Unix() - query.TimeRange.From.Unix()
// Interpolate variables in expr
expr := model.Expr
expr = strings.ReplaceAll(expr, "$__interval_ms", strconv.FormatInt(intervalMs, 10))
expr = strings.ReplaceAll(expr, "$__interval", intervalv2.FormatDuration(interval))
expr = strings.ReplaceAll(expr, "$__range_ms", strconv.FormatInt(rangeS*1000, 10))
expr = strings.ReplaceAll(expr, "$__range_s", strconv.FormatInt(rangeS, 10))
expr = strings.ReplaceAll(expr, "$__range", strconv.FormatInt(rangeS, 10)+"s")
expr = strings.ReplaceAll(expr, "$__rate_interval", intervalv2.FormatDuration(calculateRateInterval(interval, dsInfo.TimeInterval, s.intervalCalculator)))
qs = append(qs, &PrometheusQuery{
Expr: expr,
Step: step,
LegendFormat: format,
Start: start,
End: end,
RefId: queryModel.RefID,
Step: interval,
LegendFormat: model.LegendFormat,
Start: query.TimeRange.From,
End: query.TimeRange.To,
RefId: query.RefID,
})
}
@ -317,3 +340,18 @@ func ConvertAPIError(err error) error {
}
return err
}
func calculateRateInterval(interval time.Duration, scrapeInterval string, intervalCalculator intervalv2.Calculator) time.Duration {
scrape := scrapeInterval
if scrape == "" {
scrape = "15s"
}
scrapeIntervalDuration, err := intervalv2.ParseIntervalStringToTimeDuration(scrape)
if err != nil {
return time.Duration(0)
}
rateInterval := time.Duration(int(math.Max(float64(interval+scrapeIntervalDuration), float64(4)*float64(scrapeIntervalDuration))))
return rateInterval
}

235
pkg/tsdb/prometheus/prometheus_test.go
View File

@ -13,11 +13,7 @@ import (
var now = time.Now()
func TestPrometheus(t *testing.T) {
service := Service{
intervalCalculator: intervalv2.NewCalculator(),
}
func TestPrometheus_formatLeged(t *testing.T) {
t.Run("converting metric name", func(t *testing.T) {
metric := map[p.LabelName]p.LabelValue{
p.LabelName("app"): p.LabelValue("backend"),
@ -44,89 +40,252 @@ func TestPrometheus(t *testing.T) {
require.Equal(t, `http_request_total{app="backend", device="mobile"}`, formatLegend(metric, query))
})
}
func TestPrometheus_parseQuery(t *testing.T) {
service := Service{
intervalCalculator: intervalv2.NewCalculator(),
}
t.Run("parsing query model with step", func(t *testing.T) {
query := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"refId": "A"
}`)
timeRange := backend.TimeRange{
From: now,
To: now.Add(12 * time.Hour),
}
query.TimeRange = timeRange
models, err := service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{})
query := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"refId": "A"
}`, timeRange)
dsInfo := &DatasourceInfo{}
models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, time.Second*30, models[0].Step)
})
t.Run("parsing query model without step parameter", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(1 * time.Hour),
}
query := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`)
models, err := service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{})
require.NoError(t, err)
require.Equal(t, time.Minute*2, models[0].Step)
}`, timeRange)
timeRange := backend.TimeRange{
From: now,
To: now.Add(1 * time.Hour),
}
query.TimeRange = timeRange
models, err = service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{})
dsInfo := &DatasourceInfo{}
models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, time.Second*15, models[0].Step)
})
t.Run("parsing query model with high intervalFactor", func(t *testing.T) {
models, err := service.parseQuery([]backend.DataQuery{queryContext(`{
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
query := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 10,
"refId": "A"
}`)}, &DatasourceInfo{})
}`, timeRange)
dsInfo := &DatasourceInfo{}
models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, time.Minute*20, models[0].Step)
})
t.Run("parsing query model with low intervalFactor", func(t *testing.T) {
models, err := service.parseQuery([]backend.DataQuery{queryContext(`{
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
query := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`)}, &DatasourceInfo{})
}`, timeRange)
dsInfo := &DatasourceInfo{}
models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, time.Minute*2, models[0].Step)
})
t.Run("parsing query model specified scrape-interval in the data source", func(t *testing.T) {
models, err := service.parseQuery([]backend.DataQuery{queryContext(`{
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
query := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`)}, &DatasourceInfo{
}`, timeRange)
dsInfo := &DatasourceInfo{
TimeInterval: "240s",
})
}
models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, time.Minute*4, models[0].Step)
})
t.Run("parsing query model with $__interval variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
query := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__interval]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
dsInfo := &DatasourceInfo{}
models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [2m]})", models[0].Expr)
})
t.Run("parsing query model with $__interval_ms variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
query := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__interval_ms]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
dsInfo := &DatasourceInfo{}
models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [120000]})", models[0].Expr)
})
t.Run("parsing query model with $__interval_ms and $__interval variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
query := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__interval_ms]}) + rate(ALERTS{job=\"test\" [$__interval]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
dsInfo := &DatasourceInfo{}
models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [120000]}) + rate(ALERTS{job=\"test\" [2m]})", models[0].Expr)
})
t.Run("parsing query model with $__range variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
query := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__range]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
dsInfo := &DatasourceInfo{}
models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [172800s]})", models[0].Expr)
})
t.Run("parsing query model with $__range_s variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
query := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__range_s]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
dsInfo := &DatasourceInfo{}
models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [172800]})", models[0].Expr)
})
t.Run("parsing query model with $__range_ms variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
query := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__range_ms]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
dsInfo := &DatasourceInfo{}
models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [172800000]})", models[0].Expr)
})
t.Run("parsing query model with $__rate_interval variable", func(t *testing.T) {
timeRange := backend.TimeRange{
From: now,
To: now.Add(5 * time.Minute),
}
query := queryContext(`{
"expr": "rate(ALERTS{job=\"test\" [$__rate_interval]})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}`, timeRange)
dsInfo := &DatasourceInfo{}
models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, "rate(ALERTS{job=\"test\" [1m]})", models[0].Expr)
})
}
func queryContext(json string) backend.DataQuery {
timeRange := backend.TimeRange{
From: now,
To: now.Add(48 * time.Hour),
}
return backend.DataQuery{
TimeRange: timeRange,
RefID: "A",
JSON: []byte(json),
func queryContext(json string, timeRange backend.TimeRange) *backend.QueryDataRequest {
return &backend.QueryDataRequest{
Queries: []backend.DataQuery{
{
JSON: []byte(json),
TimeRange: timeRange,
RefID: "A",
},
},
}
}