api/ds/query: simplify data sources lookup for queries and expressions (#41172)

pkg/expr/graph.go

@@ -143,17 +143,30 @@ func (s *Service) buildGraph(req *Request) (*simple.DirectedGraph, error) {
 			RefID:         query.RefID,
 			TimeRange:     query.TimeRange,
 			QueryType:     query.QueryType,
-			DatasourceUID: query.DatasourceUID,
+			DatasourceUID: query.GetDatasourceUID(),
 		}
 
-		isExpr, err := rn.IsExpressionQuery()
-		if err != nil {
-			return nil, err
+		numericDSID := float64(0) // legacy
+		if rn.DatasourceUID == "" {
+			if rv, ok := rn.Query["datasourceId"]; ok {
+				if sv, ok := rv.(float64); ok {
+					if sv == DatasourceID {
+						rn.DatasourceUID = DatasourceUID
+					}
+					if sv > 0 {
+						numericDSID = sv
+					}
+				}
+			}
+		}
+
+		if rn.DatasourceUID == "" && numericDSID == 0 {
+			return nil, fmt.Errorf("missing datasource uid in query with refId %v", query.RefID)
 		}
 
 		var node Node
 
-		if isExpr {
+		if rn.IsExpressionQuery() {
 			node, err = buildCMDNode(dp, rn)
 		} else {
 			node, err = s.buildDSNode(dp, rn, req)

pkg/expr/graph_test.go

@@ -20,7 +20,7 @@ func TestServicebuildPipeLine(t *testing.T) {
 				Queries: []Query{
 					{
 						RefID:         "A",
-						DatasourceUID: DatasourceUID,
+						DatasourceUID: OldDatasourceUID,
 						JSON: json.RawMessage(`{
 							"expression": "B",
 							"reducer": "mean",
@@ -41,7 +41,7 @@ func TestServicebuildPipeLine(t *testing.T) {
 				Queries: []Query{
 					{
 						RefID:         "A",
-						DatasourceUID: DatasourceUID,
+						DatasourceUID: OldDatasourceUID,
 						JSON: json.RawMessage(`{
 								"expression": "$B",
 								"type": "math"
@@ -49,7 +49,7 @@ func TestServicebuildPipeLine(t *testing.T) {
 					},
 					{
 						RefID:         "B",
-						DatasourceUID: DatasourceUID,
+						DatasourceUID: OldDatasourceUID,
 						JSON: json.RawMessage(`{
 								"expression": "$A",
 								"type": "math"
@@ -65,7 +65,7 @@ func TestServicebuildPipeLine(t *testing.T) {
 				Queries: []Query{
 					{
 						RefID:         "A",
-						DatasourceUID: DatasourceUID,
+						DatasourceUID: OldDatasourceUID,
 						JSON: json.RawMessage(`{
 								"expression": "$A",
 								"type": "math"
@@ -81,7 +81,7 @@ func TestServicebuildPipeLine(t *testing.T) {
 				Queries: []Query{
 					{
 						RefID:         "A",
-						DatasourceUID: DatasourceUID,
+						DatasourceUID: OldDatasourceUID,
 						JSON: json.RawMessage(`{
 								"expression": "$B",
 								"type": "math"
@@ -97,7 +97,7 @@ func TestServicebuildPipeLine(t *testing.T) {
 				Queries: []Query{
 					{
 						RefID:         "A",
-						DatasourceUID: DatasourceUID,
+						DatasourceUID: OldDatasourceUID,
 						JSON: json.RawMessage(`{
 							"type": "classic_conditions",
 							"conditions": [
@@ -128,7 +128,7 @@ func TestServicebuildPipeLine(t *testing.T) {
 					},
 					{
 						RefID:         "B",
-						DatasourceUID: DatasourceUID,
+						DatasourceUID: OldDatasourceUID,
 						JSON: json.RawMessage(`{
 							"expression": "C",
 							"reducer": "mean",
@@ -149,7 +149,7 @@ func TestServicebuildPipeLine(t *testing.T) {
 				Queries: []Query{
 					{
 						RefID:         "A",
-						DatasourceUID: DatasourceUID,
+						DatasourceUID: OldDatasourceUID,
 						JSON: json.RawMessage(`{
 							"type": "classic_conditions",
 							"conditions": [
@@ -180,7 +180,7 @@ func TestServicebuildPipeLine(t *testing.T) {
 					},
 					{
 						RefID:         "B",
-						DatasourceUID: DatasourceUID,
+						DatasourceUID: OldDatasourceUID,
 						JSON: json.RawMessage(`{
 							"expression": "A",
 							"reducer": "mean",
@@ -195,33 +195,31 @@ func TestServicebuildPipeLine(t *testing.T) {
 			},
 			expectErrContains: "classic conditions may not be the input for other expressions",
 		},
-		//{
-		//	name: "Queries with new datasource ref object",
-		//	req: &Request{
-		//		Queries: []Query{
-		//			{
-		//				RefID: "A",
-		//				JSON: json.RawMessage(`{
-		//					"datasource":  {
-		//								"uid": "MyDS"
-		//							}
-		//					}`),
-		//			},
-		//			{
-		//				RefID: "B",
-		//				JSON: json.RawMessage(`{
-		//					"datasource":  {
-		//						"uid": "MyDS"
-		//					},
-		//					"expression": "A",
-		//					"reducer": "mean",
-		//					"type": "reduce"
-		//				}`),
-		//			},
-		//		},
-		//	},
-		//	expectedOrder: []string{"B", "A"},
-		//},
+		{
+			name: "Queries with new datasource ref object",
+			req: &Request{
+				Queries: []Query{
+					{
+						RefID: "A",
+						Datasource: DataSourceRef{
+							UID: DatasourceUID,
+						},
+						JSON: json.RawMessage(`{
+							"expression": "B",
+							"reducer": "mean",
+							"type": "reduce"
+						}`),
+					},
+					{
+						RefID: "B",
+						Datasource: DataSourceRef{
+							UID: "Fake",
+						},
+					},
+				},
+			},
+			expectedOrder: []string{"B", "A"},
+		},
 	}
 	s := Service{}
 	for _, tt := range tests {

pkg/expr/nodes.go

@@ -30,62 +30,19 @@ type rawNode struct {
 	Query         map[string]interface{}
 	QueryType     string
 	TimeRange     TimeRange
-	DatasourceUID string
+	DatasourceUID string // Gets populated from Either DatasourceUID or Datasource.UID
 }
 
-func (rn *rawNode) GetDatasourceUID() (string, error) {
-	if rn.DatasourceUID != "" {
-		return rn.DatasourceUID, nil
+func (rn *rawNode) IsExpressionQuery() bool {
+	if IsDataSource(rn.DatasourceUID) {
+		return true
 	}
-
-	rawDs, ok := rn.Query["datasource"]
-	if !ok {
-		return "", fmt.Errorf("no datasource property found in query model")
+	if v, ok := rn.Query["datasourceId"]; ok {
+		if v == OldDatasourceUID {
+			return true
+		}
 	}
-
-	// For old queries with string datasource prop representing data source name
-	if dsName, ok := rawDs.(string); ok {
-		return dsName, nil
-	}
-
-	dsRef, ok := rawDs.(map[string]interface{})
-	if !ok {
-		return "", fmt.Errorf("data source property is not an object nor string, got %T", rawDs)
-	}
-
-	if dsUid, ok := dsRef["uid"].(string); ok {
-		return dsUid, nil
-	}
-
-	return "", fmt.Errorf("no datasource uid found for query, got %T", rn.Query)
-}
-
-func (rn *rawNode) IsExpressionQuery() (bool, error) {
-	if rn.DatasourceUID != "" {
-		return rn.DatasourceUID == DatasourceUID, nil
-	}
-
-	rawDs, ok := rn.Query["datasource"]
-	if !ok {
-		return false, fmt.Errorf("no datasource property found in query model")
-	}
-
-	// For old queries with string datasource prop representing data source name
-	dsName, ok := rawDs.(string)
-	if ok && dsName == DatasourceName {
-		return true, nil
-	}
-
-	dsRef, ok := rawDs.(map[string]interface{})
-	if !ok {
-		return false, nil
-	}
-
-	if dsRef["uid"].(string) == DatasourceUID {
-		return true, nil
-	}
-
-	return false, nil
+	return false
 }
 
 func (rn *rawNode) GetCommandType() (c CommandType, err error) {
@@ -213,6 +170,7 @@ func (s *Service) buildDSNode(dp *simple.DirectedGraph, rn *rawNode, req *Reques
 		request:    *req,
 	}
 
+	// support old datasourceId property
 	rawDsID, ok := rn.Query["datasourceId"]
 	if ok {
 		floatDsID, ok := rawDsID.(float64)
@@ -221,11 +179,7 @@ func (s *Service) buildDSNode(dp *simple.DirectedGraph, rn *rawNode, req *Reques
 		}
 		dsNode.datasourceID = int64(floatDsID)
 	} else {
-		dsUid, err := rn.GetDatasourceUID()
-		if err != nil {
-			return nil, fmt.Errorf("neither datasourceId or datasourceUid in expression data source request for refId %v", rn.RefID)
-		}
-		dsNode.datasourceUID = dsUid
+		dsNode.datasourceUID = rn.DatasourceUID
 	}
 
 	var floatIntervalMS float64

pkg/expr/service.go

@@ -4,21 +4,34 @@ import (
 	"context"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
-// DatasourceName is the string constant used as the datasource name in requests
-// to identify it as an expression command.
-const DatasourceName = "__expr__"
+// DatasourceType is the string constant used as the datasource when the property is in Datasource.Type.
+// Type in requests is used to identify what type of data source plugin the request belongs to.
+const DatasourceType = "__expr__"
+
+// DatasourceUID is the string constant used as the datasource name in requests
+// to identify it as an expression command when use in Datasource.UID.
+const DatasourceUID = DatasourceType
 
 // DatasourceID is the fake datasource id used in requests to identify it as an
 // expression command.
 const DatasourceID = -100
 
-// DatasourceUID is the fake datasource uid used in requests to identify it as an
-// expression command.
-const DatasourceUID = "-100"
+// OldDatasourceUID is the datasource uid used in requests to identify it as an
+// expression command. It goes with the query root level datasourceUID property. It was accidentally
+// set to the Id and is now kept for backwards compatibility. The newer Datasource.UID property
+// should be used instead and should be set to "__expr__".
+const OldDatasourceUID = "-100"
+
+// IsDataSource checks if the uid points to an expression query
+func IsDataSource(uid string) bool {
+	return uid == DatasourceUID || uid == OldDatasourceUID
+}
 
 // Service is service representation for expression handling.
 type Service struct {
@@ -52,3 +65,14 @@ func (s *Service) ExecutePipeline(ctx context.Context, pipeline DataPipeline) (*
 	}
 	return res, nil
 }
+
+func DataSourceModel() *models.DataSource {
+	return &models.DataSource{
+		Id:             DatasourceID,
+		Uid:            DatasourceUID,
+		Name:           DatasourceUID,
+		Type:           DatasourceType,
+		JsonData:       simplejson.New(),
+		SecureJsonData: make(map[string][]byte),
+	}
+}

pkg/expr/transform.go

@@ -40,6 +40,9 @@ func (s *Service) WrapTransformData(ctx context.Context, query plugins.DataQuery
 	}
 
 	for _, q := range query.Queries {
+		if q.DataSource == nil {
+			return nil, fmt.Errorf("mising datasource info: " + q.RefID)
+		}
 		modelJSON, err := q.Model.MarshalJSON()
 		if err != nil {
 			return nil, err
@@ -50,6 +53,10 @@ func (s *Service) WrapTransformData(ctx context.Context, query plugins.DataQuery
 			RefID:         q.RefID,
 			MaxDataPoints: q.MaxDataPoints,
 			QueryType:     q.QueryType,
+			Datasource: DataSourceRef{
+				Type: q.DataSource.Type,
+				UID:  q.DataSource.Uid,
+			},
 			TimeRange: TimeRange{
 				From: query.TimeRange.GetFromAsTimeUTC(),
 				To:   query.TimeRange.GetToAsTimeUTC(),
@@ -72,13 +79,30 @@ type Request struct {
 type Query struct {
 	RefID         string
 	TimeRange     TimeRange
-	DatasourceUID string
+	DatasourceUID string        // deprecated, value -100 when expressions
+	Datasource    DataSourceRef `json:"datasource"`
 	JSON          json.RawMessage
 	Interval      time.Duration
 	QueryType     string
 	MaxDataPoints int64
 }
 
+type DataSourceRef struct {
+	Type string `json:"type"` // value should be __expr__
+	UID  string `json:"uid"`  // value should be __expr__
+}
+
+func (q *Query) GetDatasourceUID() string {
+	if q.DatasourceUID != "" {
+		return q.DatasourceUID // backwards compatibility gets precedence
+	}
+
+	if q.Datasource.UID != "" {
+		return q.Datasource.UID
+	}
+	return ""
+}
+
 // TimeRange is a time.Time based TimeRange.
 type TimeRange struct {
 	From time.Time

pkg/expr/translate/translate.go

@@ -259,7 +259,7 @@ func (dc *dashConditionsJSON) GetNew(orgID int64) (*ngmodels.Condition, error) {
 	ccAlertQuery := ngmodels.AlertQuery{
 		RefID:         ccRefID,
 		Model:         exprModelJSON,
-		DatasourceUID: expr.DatasourceUID,
+		DatasourceUID: expr.OldDatasourceUID,
 	}
 
 	ngCond.Data = append(ngCond.Data, ccAlertQuery)