Prometheus schematization (#63878)

* Schematize prometheus

* revert changes

* close response body

* Update report.json

* Update pkg/tsdb/prometheus/models/query.go

Co-authored-by: sam boyer <sdboyer@grafana.com>

* Use without pointers

* remove unused

* Specify query format

* Rename

* Clean up schema

* Update public/app/plugins/datasource/prometheus/dataquery.cue

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

* Update pkg/tsdb/prometheus/models/query.go

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

* Clean up tests

* Update public/app/plugins/datasource/prometheus/dataquery.cue

Co-authored-by: sam boyer <sdboyer@grafana.com>

* make gen-cue

* Add comments

* Make linter happy

* Remove editormode override

* Update

---------

Co-authored-by: sam boyer <sdboyer@grafana.com>
Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

pkg/tsdb/prometheus/kinds/dataquery/types_dataquery_gen.go

@@ -0,0 +1,89 @@
+// Code generated - EDITING IS FUTILE. DO NOT EDIT.
+//
+// Generated by:
+//     public/app/plugins/gen.go
+// Using jennies:
+//     PluginGoTypesJenny
+//
+// Run 'make gen-cue' from repository root to regenerate.
+
+package dataquery
+
+// Defines values for PromQueryFormat.
+const (
+	PromQueryFormatHeatmap    PromQueryFormat = "heatmap"
+	PromQueryFormatTable      PromQueryFormat = "table"
+	PromQueryFormatTimeSeries PromQueryFormat = "time_series"
+)
+
+// Defines values for EditorMode.
+const (
+	EditorModeBuilder EditorMode = "builder"
+	EditorModeCode    EditorMode = "code"
+)
+
+// Defines values for Format.
+const (
+	FormatHeatmap    Format = "heatmap"
+	FormatTable      Format = "table"
+	FormatTimeSeries Format = "time_series"
+)
+
+// Defines values for QueryEditorMode.
+const (
+	QueryEditorModeBuilder QueryEditorMode = "builder"
+	QueryEditorModeCode    QueryEditorMode = "code"
+)
+
+// PromQueryFormat defines model for PromQueryFormat.
+type PromQueryFormat string
+
+// PrometheusDataQuery defines model for PrometheusDataQuery.
+type PrometheusDataQuery struct {
+	// For mixed data sources the selected datasource is on the query level.
+	// For non mixed scenarios this is undefined.
+	// TODO find a better way to do this ^ that's friendly to schema
+	// TODO this shouldn't be unknown but DataSourceRef | null
+	Datasource *interface{} `json:"datasource,omitempty"`
+
+	// Specifies which editor is being used to prepare the query. It can be "code" or "builder"
+	EditorMode *EditorMode `json:"editorMode,omitempty"`
+
+	// Execute an additional query to identify interesting raw samples relevant for the given expr
+	Exemplar *bool `json:"exemplar,omitempty"`
+
+	// The actual expression/query that will be evaluated by Prometheus
+	Expr string `json:"expr"`
+
+	// Query format to determine how to display data points in panel. It can be "time_series", "table", "heatmap"
+	Format *Format `json:"format,omitempty"`
+
+	// Hide true if query is disabled (ie should not be returned to the dashboard)
+	// Note this does not always imply that the query should not be executed since
+	// the results from a hidden query may be used as the input to other queries (SSE etc)
+	Hide *bool `json:"hide,omitempty"`
+
+	// Returns only the latest value that Prometheus has scraped for the requested time series
+	Instant *bool `json:"instant,omitempty"`
+
+	// Specify the query flavor
+	// TODO make this required and give it a default
+	QueryType *string `json:"queryType,omitempty"`
+
+	// Returns a Range vector, comprised of a set of time series containing a range of data points over time for each time series
+	Range *bool `json:"range,omitempty"`
+
+	// A unique identifier for the query within the list of targets.
+	// In server side expressions, the refId is used as a variable name to identify results.
+	// By default, the UI will assign A->Z; however setting meaningful names may be useful.
+	RefId string `json:"refId"`
+}
+
+// Specifies which editor is being used to prepare the query. It can be "code" or "builder"
+type EditorMode string
+
+// Query format to determine how to display data points in panel. It can be "time_series", "table", "heatmap"
+type Format string
+
+// QueryEditorMode defines model for QueryEditorMode.
+type QueryEditorMode string

pkg/tsdb/prometheus/models/query.go

@@ -8,8 +8,8 @@ import (
 	"time"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
-
 	"github.com/grafana/grafana/pkg/tsdb/intervalv2"
+	"github.com/grafana/grafana/pkg/tsdb/prometheus/kinds/dataquery"
 )
 
 // Internal interval and range variables
@@ -45,16 +45,14 @@ const (
 var safeResolution = 11000
 
 type QueryModel struct {
-	Expr           string `json:"expr"`
-	LegendFormat   string `json:"legendFormat"`
-	Interval       string `json:"interval"`
-	IntervalMS     int64  `json:"intervalMS"`
-	StepMode       string `json:"stepMode"`
-	RangeQuery     bool   `json:"range"`
-	InstantQuery   bool   `json:"instant"`
-	ExemplarQuery  bool   `json:"exemplar"`
-	IntervalFactor int64  `json:"intervalFactor"`
-	UtcOffsetSec   int64  `json:"utcOffsetSec"`
+	dataquery.PrometheusDataQuery
+	// The following properties may be part of the request payload, however they are not saved in panel JSON
+	// Timezone offset to align start & end time on backend
+	UtcOffsetSec   int64  `json:"utcOffsetSec,omitempty"`
+	LegendFormat   string `json:"legendFormat,omitempty"`
+	Interval       string `json:"interval,omitempty"`
+	IntervalMs     int64  `json:"intervalMs,omitempty"`
+	IntervalFactor int64  `json:"intervalFactor,omitempty"`
 }
 
 type TimeRange struct {
@@ -82,23 +80,36 @@ func Parse(query backend.DataQuery, timeInterval string, intervalCalculator inte
 		return nil, err
 	}
 
-	//Final interval value
-	interval, err := calculatePrometheusInterval(model, timeInterval, query, intervalCalculator)
+	// Final interval value
+	interval, err := calculatePrometheusInterval(model.Interval, timeInterval, model.IntervalMs, model.IntervalFactor, query, intervalCalculator)
 	if err != nil {
 		return nil, err
 	}
 
 	// Interpolate variables in expr
 	timeRange := query.TimeRange.To.Sub(query.TimeRange.From)
-	expr := interpolateVariables(model, interval, timeRange, intervalCalculator, timeInterval)
-	rangeQuery := model.RangeQuery
-	if !model.InstantQuery && !model.RangeQuery {
+	expr := interpolateVariables(model.Expr, model.Interval, interval, timeRange, intervalCalculator, timeInterval)
+	var rangeQuery, instantQuery bool
+	if model.Instant == nil {
+		instantQuery = false
+	} else {
+		instantQuery = *model.Instant
+	}
+	if model.Range == nil {
+		rangeQuery = false
+	} else {
+		rangeQuery = *model.Range
+	}
+	if !instantQuery && !rangeQuery {
 		// In older dashboards, we were not setting range query param and !range && !instant was run as range query
 		rangeQuery = true
 	}
 
 	// We never want to run exemplar query for alerting
-	exemplarQuery := model.ExemplarQuery
+	exemplarQuery := false
+	if model.Exemplar != nil {
+		exemplarQuery = *model.Exemplar
+	}
 	if fromAlert {
 		exemplarQuery = false
 	}
@@ -110,7 +121,7 @@ func Parse(query backend.DataQuery, timeInterval string, intervalCalculator inte
 		Start:         query.TimeRange.From,
 		End:           query.TimeRange.To,
 		RefId:         query.RefID,
-		InstantQuery:  model.InstantQuery,
+		InstantQuery:  instantQuery,
 		RangeQuery:    rangeQuery,
 		ExemplarQuery: exemplarQuery,
 		UtcOffsetSec:  model.UtcOffsetSec,
@@ -139,15 +150,18 @@ func (query *Query) TimeRange() TimeRange {
 	}
 }
 
-func calculatePrometheusInterval(model *QueryModel, timeInterval string, query backend.DataQuery, intervalCalculator intervalv2.Calculator) (time.Duration, error) {
-	queryInterval := model.Interval
-
-	//If we are using variable for interval/step, we will replace it with calculated interval
+func calculatePrometheusInterval(
+	queryInterval, timeInterval string,
+	intervalMs, intervalFactor int64,
+	query backend.DataQuery,
+	intervalCalculator intervalv2.Calculator,
+) (time.Duration, error) {
+	// If we are using variable for interval/step, we will replace it with calculated interval
 	if isVariableInterval(queryInterval) {
 		queryInterval = ""
 	}
 
-	minInterval, err := intervalv2.GetIntervalFrom(timeInterval, queryInterval, model.IntervalMS, 15*time.Second)
+	minInterval, err := intervalv2.GetIntervalFrom(timeInterval, queryInterval, intervalMs, 15*time.Second)
 	if err != nil {
 		return time.Duration(0), err
 	}
@@ -159,19 +173,23 @@ func calculatePrometheusInterval(model *QueryModel, timeInterval string, query b
 		adjustedInterval = calculatedInterval.Value
 	}
 
-	if model.Interval == varRateInterval || model.Interval == varRateIntervalAlt {
+	if queryInterval == varRateInterval || queryInterval == varRateIntervalAlt {
 		// Rate interval is final and is not affected by resolution
 		return calculateRateInterval(adjustedInterval, timeInterval, intervalCalculator), nil
 	} else {
-		intervalFactor := model.IntervalFactor
-		if intervalFactor == 0 {
-			intervalFactor = 1
+		queryIntervalFactor := intervalFactor
+		if queryIntervalFactor == 0 {
+			queryIntervalFactor = 1
 		}
-		return time.Duration(int64(adjustedInterval) * intervalFactor), nil
+		return time.Duration(int64(adjustedInterval) * queryIntervalFactor), nil
 	}
 }
 
-func calculateRateInterval(interval time.Duration, scrapeInterval string, intervalCalculator intervalv2.Calculator) time.Duration {
+func calculateRateInterval(
+	interval time.Duration,
+	scrapeInterval string,
+	intervalCalculator intervalv2.Calculator,
+) time.Duration {
 	scrape := scrapeInterval
 	if scrape == "" {
 		scrape = "15s"
@@ -186,13 +204,14 @@ func calculateRateInterval(interval time.Duration, scrapeInterval string, interv
 	return rateInterval
 }
 
-func interpolateVariables(model *QueryModel, interval time.Duration, timeRange time.Duration, intervalCalculator intervalv2.Calculator, timeInterval string) string {
-	expr := model.Expr
+func interpolateVariables(expr, queryInterval string, interval time.Duration,
+	timeRange time.Duration,
+	intervalCalculator intervalv2.Calculator, timeInterval string) string {
 	rangeMs := timeRange.Milliseconds()
 	rangeSRounded := int64(math.Round(float64(rangeMs) / 1000.0))
 
 	var rateInterval time.Duration
-	if model.Interval == varRateInterval || model.Interval == varRateIntervalAlt {
+	if queryInterval == varRateInterval || queryInterval == varRateIntervalAlt {
 		rateInterval = interval
 	} else {
 		rateInterval = calculateRateInterval(interval, timeInterval, intervalCalculator)
@@ -219,7 +238,7 @@ func isVariableInterval(interval string) bool {
 	if interval == varInterval || interval == varIntervalMs || interval == varRateInterval {
 		return true
 	}
-	//Repetitive code, we should have functionality to unify these
+	// Repetitive code, we should have functionality to unify these
 	if interval == varIntervalAlt || interval == varIntervalMsAlt || interval == varRateIntervalAlt {
 		return true
 	}

pkg/tsdb/prometheus/models/query_test.go

@@ -376,6 +376,7 @@ func TestParse(t *testing.T) {
 			"format": "time_series",
 			"intervalFactor": 1,
 			"interval": "$__rate_interval",
+			"intervalMs": 60000,
 			"refId": "A"
 		}`, timeRange)
 

pkg/tsdb/prometheus/querydata/framing_bench_test.go

@@ -15,6 +15,8 @@ import (
 	"time"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana/pkg/kindsys"
+	"github.com/grafana/grafana/pkg/tsdb/prometheus/kinds/dataquery"
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/tsdb/prometheus/models"
@@ -121,8 +123,10 @@ func createJsonTestData(start int64, step int64, timestampCount int, seriesCount
 	bytes := []byte(fmt.Sprintf(`{"status":"success","data":{"resultType":"matrix","result":[%v]}}`, strings.Join(allSeries, ",")))
 
 	qm := models.QueryModel{
-		RangeQuery: true,
-		Expr:       "test",
+		PrometheusDataQuery: dataquery.PrometheusDataQuery{
+			Range: kindsys.Ptr(true),
+			Expr:  "test",
+		},
 	}
 
 	data, err := json.Marshal(&qm)

pkg/tsdb/prometheus/querydata/framing_test.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/experimental"
+	"github.com/grafana/grafana/pkg/tsdb/prometheus/kinds/dataquery"
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/tsdb/prometheus/models"
@@ -115,12 +116,14 @@ func loadStoredQuery(fileName string) (*backend.QueryDataRequest, error) {
 	}
 
 	qm := models.QueryModel{
-		RangeQuery:    sq.RangeQuery,
-		ExemplarQuery: sq.ExemplarQuery,
-		Expr:          sq.Expr,
-		Interval:      fmt.Sprintf("%ds", sq.Step),
-		IntervalMS:    sq.Step * 1000,
-		LegendFormat:  sq.LegendFormat,
+		PrometheusDataQuery: dataquery.PrometheusDataQuery{
+			Range:    &sq.RangeQuery,
+			Exemplar: &sq.ExemplarQuery,
+			Expr:     sq.Expr,
+		},
+		Interval:     fmt.Sprintf("%ds", sq.Step),
+		IntervalMs:   sq.Step * 1000,
+		LegendFormat: sq.LegendFormat,
 	}
 
 	data, err := json.Marshal(&qm)

pkg/tsdb/prometheus/querydata/request_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"math"
 	"net/http"
@@ -13,6 +14,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/tsdb/prometheus/kinds/dataquery"
 	apiv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	p "github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -20,6 +22,7 @@ import (
 	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log/logtest"
 	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/grafana/grafana/pkg/kindsys"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb/prometheus/client"
 	"github.com/grafana/grafana/pkg/tsdb/prometheus/models"
@@ -65,9 +68,11 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) {
 		require.NoError(t, err)
 
 		qm := models.QueryModel{
-			LegendFormat:  "legend {{app}}",
-			UtcOffsetSec:  0,
-			ExemplarQuery: true,
+			LegendFormat: "legend {{app}}",
+			UtcOffsetSec: 0,
+			PrometheusDataQuery: dataquery.PrometheusDataQuery{
+				Exemplar: kindsys.Ptr(true),
+			},
 		}
 		b, err := json.Marshal(&qm)
 		require.NoError(t, err)
@@ -112,7 +117,9 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) {
 		qm := models.QueryModel{
 			LegendFormat: "legend {{app}}",
 			UtcOffsetSec: 0,
-			RangeQuery:   true,
+			PrometheusDataQuery: dataquery.PrometheusDataQuery{
+				Range: kindsys.Ptr(true),
+			},
 		}
 		b, err := json.Marshal(&qm)
 		require.NoError(t, err)
@@ -159,7 +166,9 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) {
 		qm := models.QueryModel{
 			LegendFormat: "",
 			UtcOffsetSec: 0,
-			RangeQuery:   true,
+			PrometheusDataQuery: dataquery.PrometheusDataQuery{
+				Range: kindsys.Ptr(true),
+			},
 		}
 		b, err := json.Marshal(&qm)
 		require.NoError(t, err)
@@ -202,7 +211,9 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) {
 		qm := models.QueryModel{
 			LegendFormat: "",
 			UtcOffsetSec: 0,
-			RangeQuery:   true,
+			PrometheusDataQuery: dataquery.PrometheusDataQuery{
+				Range: kindsys.Ptr(true),
+			},
 		}
 		b, err := json.Marshal(&qm)
 		require.NoError(t, err)
@@ -243,7 +254,9 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) {
 		qm := models.QueryModel{
 			LegendFormat: "",
 			UtcOffsetSec: 0,
-			RangeQuery:   true,
+			PrometheusDataQuery: dataquery.PrometheusDataQuery{
+				Range: kindsys.Ptr(true),
+			},
 		}
 		b, err := json.Marshal(&qm)
 		require.NoError(t, err)
@@ -278,7 +291,9 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) {
 		qm := models.QueryModel{
 			LegendFormat: "legend {{app}}",
 			UtcOffsetSec: 0,
-			InstantQuery: true,
+			PrometheusDataQuery: dataquery.PrometheusDataQuery{
+				Instant: kindsys.Ptr(true),
+			},
 		}
 		b, err := json.Marshal(&qm)
 		require.NoError(t, err)
@@ -317,7 +332,9 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) {
 		qm := models.QueryModel{
 			LegendFormat: "",
 			UtcOffsetSec: 0,
-			InstantQuery: true,
+			PrometheusDataQuery: dataquery.PrometheusDataQuery{
+				Instant: kindsys.Ptr(true),
+			},
 		}
 		b, err := json.Marshal(&qm)
 		require.NoError(t, err)
@@ -354,20 +371,20 @@ func executeWithHeaders(tctx *testContext, query backend.DataQuery, qr interface
 	}
 
 	promRes, err := toAPIResponse(qr)
+	defer func() {
+		if err := promRes.Body.Close(); err != nil {
+			fmt.Println(fmt.Errorf("response body close error: %v", err))
+		}
+	}()
 	if err != nil {
 		return nil, err
 	}
-
 	tctx.httpProvider.setResponse(promRes)
 
 	res, err := tctx.queryData.Execute(context.Background(), &req)
-	errClose := promRes.Body.Close()
 	if err != nil {
 		return nil, err
 	}
-	if errClose != nil {
-		return nil, errClose
-	}
 
 	return res.Responses[req.Queries[0].RefID].Frames, nil
 }