Prometheus: Add interpolation for built-in-time variables to backend (#39051)

* Add grafana variable interpolation to backend

* Remove newlines

pkg/tsdb/intervalv2/intervalv2.go

@@ -82,31 +82,40 @@ func (ic *intervalCalculator) CalculateSafeInterval(timerange backend.TimeRange,
 // queryIntervalMS is a pre-calculated numeric representation of the query interval in milliseconds.
 func GetIntervalFrom(dsInterval, queryInterval string, queryIntervalMS int64, defaultInterval time.Duration) (time.Duration, error) {
 	// Apparently we are setting default value of queryInterval to 0s now
-	if queryInterval == "0s" {
+	interval := queryInterval
-		queryInterval = ""
+	if interval == "0s" {
+		interval = ""
 	}
-
+	if interval == "" {
-	if queryInterval == "" {
 		if queryIntervalMS != 0 {
 			return time.Duration(queryIntervalMS) * time.Millisecond, nil
 		}
 	}
-	interval := queryInterval
+	if interval == "" && dsInterval != "" {
-	if queryInterval == "" && dsInterval != "" {
 		interval = dsInterval
 	}
 	if interval == "" {
 		return defaultInterval, nil
 	}
-	interval = strings.Replace(strings.Replace(interval, "<", "", 1), ">", "", 1)
+
-	isPureNum, err := regexp.MatchString(`^\d+$`, interval)
+	parsedInterval, err := ParseIntervalStringToTimeDuration(interval)
+	if err != nil {
+		return time.Duration(0), err
+	}
+
+	return parsedInterval, nil
+}
+
+func ParseIntervalStringToTimeDuration(interval string) (time.Duration, error) {
+	formattedInterval := strings.Replace(strings.Replace(interval, "<", "", 1), ">", "", 1)
+	isPureNum, err := regexp.MatchString(`^\d+$`, formattedInterval)
 	if err != nil {
 		return time.Duration(0), err
 	}
 	if isPureNum {
-		interval += "s"
+		formattedInterval += "s"
 	}
-	parsedInterval, err := time.ParseDuration(interval)
+	parsedInterval, err := time.ParseDuration(formattedInterval)
 	if err != nil {
 		return time.Duration(0), err
 	}

pkg/tsdb/prometheus/prometheus.go

@@ -5,8 +5,10 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math"
 	"net/http"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
@@ -15,7 +17,6 @@ import (
 	sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
-	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/plugins/backendplugin"
@@ -41,6 +42,17 @@ type DatasourceInfo struct {
 	TimeInterval   string
 }
 
+type QueryModel struct {
+	Expr           string `json:"expr"`
+	LegendFormat   string `json:"legendFormat"`
+	Interval       string `json:"interval"`
+	IntervalMS     int64  `json:"intervalMS"`
+	StepMode       string `json:"stepMode"`
+	RangeQuery     bool   `json:"range"`
+	InstantQuery   bool   `json:"instant"`
+	IntervalFactor int64  `json:"intervalFactor"`
+}
+
 type Service struct {
 	httpClientProvider httpclient.Provider
 	intervalCalculator intervalv2.Calculator
@@ -130,7 +142,7 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
 		Responses: backend.Responses{},
 	}
 
-	queries, err := s.parseQuery(req.Queries, dsInfo)
+	queries, err := s.parseQuery(req, dsInfo)
 	if err != nil {
 		return &result, err
 	}
@@ -225,48 +237,59 @@ func formatLegend(metric model.Metric, query *PrometheusQuery) string {
 	return string(result)
 }
 
-func (s *Service) parseQuery(queries []backend.DataQuery, dsInfo *DatasourceInfo) ([]*PrometheusQuery, error) {
+func (s *Service) parseQuery(queryContext *backend.QueryDataRequest, dsInfo *DatasourceInfo) ([]*PrometheusQuery, error) {
 	qs := []*PrometheusQuery{}
-	for _, queryModel := range queries {
+	for _, query := range queryContext.Queries {
-		jsonModel, err := simplejson.NewJson(queryModel.JSON)
+		model := &QueryModel{}
-		if err != nil {
+		err := json.Unmarshal(query.JSON, model)
-			return nil, err
-		}
-		expr, err := jsonModel.Get("expr").String()
 		if err != nil {
 			return nil, err
 		}
 
-		format := jsonModel.Get("legendFormat").MustString("")
+		//Calculate interval
-
+		queryInterval := model.Interval
-		start := queryModel.TimeRange.From
+		//If we are using variable or interval/step, we will replace it with calculated interval
-		end := queryModel.TimeRange.To
+		if queryInterval == "$__interval" || queryInterval == "$__interval_ms" {
-		queryInterval := jsonModel.Get("interval").MustString("")
+			queryInterval = ""
-
+		}
-		minInterval, err := intervalv2.GetIntervalFrom(dsInfo.TimeInterval, queryInterval, 0, 15*time.Second)
+		minInterval, err := intervalv2.GetIntervalFrom(dsInfo.TimeInterval, queryInterval, model.IntervalMS, 15*time.Second)
 		if err != nil {
 			return nil, err
 		}
 
-		calculatedInterval := s.intervalCalculator.Calculate(queries[0].TimeRange, minInterval)
+		calculatedInterval := s.intervalCalculator.Calculate(query.TimeRange, minInterval)
-
+		safeInterval := s.intervalCalculator.CalculateSafeInterval(query.TimeRange, int64(safeRes))
-		safeInterval := s.intervalCalculator.CalculateSafeInterval(queries[0].TimeRange, int64(safeRes))
 
 		adjustedInterval := safeInterval.Value
 		if calculatedInterval.Value > safeInterval.Value {
 			adjustedInterval = calculatedInterval.Value
 		}
 
-		intervalFactor := jsonModel.Get("intervalFactor").MustInt64(1)
+		intervalFactor := model.IntervalFactor
-		step := time.Duration(int64(adjustedInterval) * intervalFactor)
+		if intervalFactor == 0 {
+			intervalFactor = 1
+		}
+
+		interval := time.Duration(int64(adjustedInterval) * intervalFactor)
+		intervalMs := int64(interval / time.Millisecond)
+		rangeS := query.TimeRange.To.Unix() - query.TimeRange.From.Unix()
+
+		// Interpolate variables in expr
+		expr := model.Expr
+		expr = strings.ReplaceAll(expr, "$__interval_ms", strconv.FormatInt(intervalMs, 10))
+		expr = strings.ReplaceAll(expr, "$__interval", intervalv2.FormatDuration(interval))
+		expr = strings.ReplaceAll(expr, "$__range_ms", strconv.FormatInt(rangeS*1000, 10))
+		expr = strings.ReplaceAll(expr, "$__range_s", strconv.FormatInt(rangeS, 10))
+		expr = strings.ReplaceAll(expr, "$__range", strconv.FormatInt(rangeS, 10)+"s")
+		expr = strings.ReplaceAll(expr, "$__rate_interval", intervalv2.FormatDuration(calculateRateInterval(interval, dsInfo.TimeInterval, s.intervalCalculator)))
 
 		qs = append(qs, &PrometheusQuery{
 			Expr:         expr,
-			Step:         step,
+			Step:         interval,
-			LegendFormat: format,
+			LegendFormat: model.LegendFormat,
-			Start:        start,
+			Start:        query.TimeRange.From,
-			End:          end,
+			End:          query.TimeRange.To,
-			RefId:        queryModel.RefID,
+			RefId:        query.RefID,
 		})
 	}
 
@@ -317,3 +340,18 @@ func ConvertAPIError(err error) error {
 	}
 	return err
 }
+
+func calculateRateInterval(interval time.Duration, scrapeInterval string, intervalCalculator intervalv2.Calculator) time.Duration {
+	scrape := scrapeInterval
+	if scrape == "" {
+		scrape = "15s"
+	}
+
+	scrapeIntervalDuration, err := intervalv2.ParseIntervalStringToTimeDuration(scrape)
+	if err != nil {
+		return time.Duration(0)
+	}
+
+	rateInterval := time.Duration(int(math.Max(float64(interval+scrapeIntervalDuration), float64(4)*float64(scrapeIntervalDuration))))
+	return rateInterval
+}

pkg/tsdb/prometheus/prometheus_test.go

@@ -13,11 +13,7 @@ import (
 
 var now = time.Now()
 
-func TestPrometheus(t *testing.T) {
+func TestPrometheus_formatLeged(t *testing.T) {
-	service := Service{
-		intervalCalculator: intervalv2.NewCalculator(),
-	}
-
 	t.Run("converting metric name", func(t *testing.T) {
 		metric := map[p.LabelName]p.LabelValue{
 			p.LabelName("app"):    p.LabelValue("backend"),
@@ -44,89 +40,252 @@ func TestPrometheus(t *testing.T) {
 
 		require.Equal(t, `http_request_total{app="backend", device="mobile"}`, formatLegend(metric, query))
 	})
+}
+
+func TestPrometheus_parseQuery(t *testing.T) {
+	service := Service{
+		intervalCalculator: intervalv2.NewCalculator(),
+	}
 
 	t.Run("parsing query model with step", func(t *testing.T) {
-		query := queryContext(`{
-			"expr": "go_goroutines",
-			"format": "time_series",
-			"refId": "A"
-		}`)
 		timeRange := backend.TimeRange{
 			From: now,
 			To:   now.Add(12 * time.Hour),
 		}
-		query.TimeRange = timeRange
+
-		models, err := service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{})
+		query := queryContext(`{
+			"expr": "go_goroutines",
+			"format": "time_series",
+			"refId": "A"
+		}`, timeRange)
+
+		dsInfo := &DatasourceInfo{}
+		models, err := service.parseQuery(query, dsInfo)
 		require.NoError(t, err)
 		require.Equal(t, time.Second*30, models[0].Step)
 	})
 
 	t.Run("parsing query model without step parameter", func(t *testing.T) {
+		timeRange := backend.TimeRange{
+			From: now,
+			To:   now.Add(1 * time.Hour),
+		}
+
 		query := queryContext(`{
 			"expr": "go_goroutines",
 			"format": "time_series",
 			"intervalFactor": 1,
 			"refId": "A"
-		}`)
+		}`, timeRange)
-		models, err := service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{})
-		require.NoError(t, err)
-		require.Equal(t, time.Minute*2, models[0].Step)
 
-		timeRange := backend.TimeRange{
+		dsInfo := &DatasourceInfo{}
-			From: now,
+		models, err := service.parseQuery(query, dsInfo)
-			To:   now.Add(1 * time.Hour),
-		}
-		query.TimeRange = timeRange
-		models, err = service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{})
 		require.NoError(t, err)
 		require.Equal(t, time.Second*15, models[0].Step)
 	})
 
 	t.Run("parsing query model with high intervalFactor", func(t *testing.T) {
-		models, err := service.parseQuery([]backend.DataQuery{queryContext(`{
+		timeRange := backend.TimeRange{
+			From: now,
+			To:   now.Add(48 * time.Hour),
+		}
+
+		query := queryContext(`{
 			"expr": "go_goroutines",
 			"format": "time_series",
 			"intervalFactor": 10,
 			"refId": "A"
-		}`)}, &DatasourceInfo{})
+		}`, timeRange)
+
+		dsInfo := &DatasourceInfo{}
+		models, err := service.parseQuery(query, dsInfo)
 		require.NoError(t, err)
 		require.Equal(t, time.Minute*20, models[0].Step)
 	})
 
 	t.Run("parsing query model with low intervalFactor", func(t *testing.T) {
-		models, err := service.parseQuery([]backend.DataQuery{queryContext(`{
+		timeRange := backend.TimeRange{
+			From: now,
+			To:   now.Add(48 * time.Hour),
+		}
+
+		query := queryContext(`{
 			"expr": "go_goroutines",
 			"format": "time_series",
 			"intervalFactor": 1,
 			"refId": "A"
-		}`)}, &DatasourceInfo{})
+		}`, timeRange)
+
+		dsInfo := &DatasourceInfo{}
+		models, err := service.parseQuery(query, dsInfo)
 		require.NoError(t, err)
 		require.Equal(t, time.Minute*2, models[0].Step)
 	})
 
 	t.Run("parsing query model specified scrape-interval in the data source", func(t *testing.T) {
-		models, err := service.parseQuery([]backend.DataQuery{queryContext(`{
-			"expr": "go_goroutines",
-			"format": "time_series",
-			"intervalFactor": 1,
-			"refId": "A"
-		}`)}, &DatasourceInfo{
-			TimeInterval: "240s",
-		})
-		require.NoError(t, err)
-		require.Equal(t, time.Minute*4, models[0].Step)
-	})
-}
-
-func queryContext(json string) backend.DataQuery {
 		timeRange := backend.TimeRange{
 			From: now,
 			To:   now.Add(48 * time.Hour),
 		}
-	return backend.DataQuery{
+
+		query := queryContext(`{
+			"expr": "go_goroutines",
+			"format": "time_series",
+			"intervalFactor": 1,
+			"refId": "A"
+		}`, timeRange)
+
+		dsInfo := &DatasourceInfo{
+			TimeInterval: "240s",
+		}
+		models, err := service.parseQuery(query, dsInfo)
+		require.NoError(t, err)
+		require.Equal(t, time.Minute*4, models[0].Step)
+	})
+
+	t.Run("parsing query model with $__interval variable", func(t *testing.T) {
+		timeRange := backend.TimeRange{
+			From: now,
+			To:   now.Add(48 * time.Hour),
+		}
+
+		query := queryContext(`{
+			"expr": "rate(ALERTS{job=\"test\" [$__interval]})",
+			"format": "time_series",
+			"intervalFactor": 1,
+			"refId": "A"
+		}`, timeRange)
+
+		dsInfo := &DatasourceInfo{}
+		models, err := service.parseQuery(query, dsInfo)
+		require.NoError(t, err)
+		require.Equal(t, "rate(ALERTS{job=\"test\" [2m]})", models[0].Expr)
+	})
+
+	t.Run("parsing query model with $__interval_ms variable", func(t *testing.T) {
+		timeRange := backend.TimeRange{
+			From: now,
+			To:   now.Add(48 * time.Hour),
+		}
+
+		query := queryContext(`{
+			"expr": "rate(ALERTS{job=\"test\" [$__interval_ms]})",
+			"format": "time_series",
+			"intervalFactor": 1,
+			"refId": "A"
+		}`, timeRange)
+
+		dsInfo := &DatasourceInfo{}
+		models, err := service.parseQuery(query, dsInfo)
+		require.NoError(t, err)
+		require.Equal(t, "rate(ALERTS{job=\"test\" [120000]})", models[0].Expr)
+	})
+
+	t.Run("parsing query model with $__interval_ms and $__interval variable", func(t *testing.T) {
+		timeRange := backend.TimeRange{
+			From: now,
+			To:   now.Add(48 * time.Hour),
+		}
+
+		query := queryContext(`{
+			"expr": "rate(ALERTS{job=\"test\" [$__interval_ms]}) + rate(ALERTS{job=\"test\" [$__interval]})",
+			"format": "time_series",
+			"intervalFactor": 1,
+			"refId": "A"
+		}`, timeRange)
+
+		dsInfo := &DatasourceInfo{}
+		models, err := service.parseQuery(query, dsInfo)
+		require.NoError(t, err)
+		require.Equal(t, "rate(ALERTS{job=\"test\" [120000]}) + rate(ALERTS{job=\"test\" [2m]})", models[0].Expr)
+	})
+
+	t.Run("parsing query model with $__range variable", func(t *testing.T) {
+		timeRange := backend.TimeRange{
+			From: now,
+			To:   now.Add(48 * time.Hour),
+		}
+
+		query := queryContext(`{
+			"expr": "rate(ALERTS{job=\"test\" [$__range]})",
+			"format": "time_series",
+			"intervalFactor": 1,
+			"refId": "A"
+		}`, timeRange)
+
+		dsInfo := &DatasourceInfo{}
+		models, err := service.parseQuery(query, dsInfo)
+		require.NoError(t, err)
+		require.Equal(t, "rate(ALERTS{job=\"test\" [172800s]})", models[0].Expr)
+	})
+
+	t.Run("parsing query model with $__range_s variable", func(t *testing.T) {
+		timeRange := backend.TimeRange{
+			From: now,
+			To:   now.Add(48 * time.Hour),
+		}
+
+		query := queryContext(`{
+			"expr": "rate(ALERTS{job=\"test\" [$__range_s]})",
+			"format": "time_series",
+			"intervalFactor": 1,
+			"refId": "A"
+		}`, timeRange)
+
+		dsInfo := &DatasourceInfo{}
+		models, err := service.parseQuery(query, dsInfo)
+		require.NoError(t, err)
+		require.Equal(t, "rate(ALERTS{job=\"test\" [172800]})", models[0].Expr)
+	})
+
+	t.Run("parsing query model with $__range_ms variable", func(t *testing.T) {
+		timeRange := backend.TimeRange{
+			From: now,
+			To:   now.Add(48 * time.Hour),
+		}
+
+		query := queryContext(`{
+			"expr": "rate(ALERTS{job=\"test\" [$__range_ms]})",
+			"format": "time_series",
+			"intervalFactor": 1,
+			"refId": "A"
+		}`, timeRange)
+
+		dsInfo := &DatasourceInfo{}
+		models, err := service.parseQuery(query, dsInfo)
+		require.NoError(t, err)
+		require.Equal(t, "rate(ALERTS{job=\"test\" [172800000]})", models[0].Expr)
+	})
+
+	t.Run("parsing query model with $__rate_interval variable", func(t *testing.T) {
+		timeRange := backend.TimeRange{
+			From: now,
+			To:   now.Add(5 * time.Minute),
+		}
+
+		query := queryContext(`{
+			"expr": "rate(ALERTS{job=\"test\" [$__rate_interval]})",
+			"format": "time_series",
+			"intervalFactor": 1,
+			"refId": "A"
+		}`, timeRange)
+
+		dsInfo := &DatasourceInfo{}
+		models, err := service.parseQuery(query, dsInfo)
+		require.NoError(t, err)
+		require.Equal(t, "rate(ALERTS{job=\"test\" [1m]})", models[0].Expr)
+	})
+}
+
+func queryContext(json string, timeRange backend.TimeRange) *backend.QueryDataRequest {
+	return &backend.QueryDataRequest{
+		Queries: []backend.DataQuery{
+			{
+				JSON:      []byte(json),
 				TimeRange: timeRange,
 				RefID:     "A",
-		JSON:      []byte(json),
+			},
+		},
 	}
 }