Public Dashboards: Add Expressions Support (#54336)

Adds support for expressions with public dashboards

pkg/services/publicdashboards/public_dashboard_service_mock.go

@@ -6,10 +6,10 @@ import (
 	context "context"
 	testing "testing"
 
-	mock "github.com/stretchr/testify/mock"
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	models "github.com/grafana/grafana/pkg/models"
 	publicdashboardsmodels "github.com/grafana/grafana/pkg/services/publicdashboards/models"
-	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	mock "github.com/stretchr/testify/mock"
 
 	user "github.com/grafana/grafana/pkg/services/user"
 )

pkg/services/publicdashboards/service/service.go

@@ -16,6 +16,7 @@ import (
 	. "github.com/grafana/grafana/pkg/services/publicdashboards/models"
 	"github.com/grafana/grafana/pkg/services/publicdashboards/validation"
 	"github.com/grafana/grafana/pkg/services/query"
+	queryModels "github.com/grafana/grafana/pkg/services/query/models"
 	"github.com/grafana/grafana/pkg/services/user"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb/intervalv2"
@@ -222,7 +223,7 @@ func (pd *PublicDashboardServiceImpl) GetQueryDataResponse(ctx context.Context,
 // dashboard and returns a metrics request to be sent to query backend
 func (pd *PublicDashboardServiceImpl) buildPublicDashboardMetricRequest(ctx context.Context, dashboard *models.Dashboard, publicDashboard *PublicDashboard, panelId int64, reqDTO *PublicDashboardQueryDTO) (dtos.MetricRequest, error) {
 	// group queries by panel
-	queriesByPanel := models.GroupQueriesByPanelId(dashboard.Data)
+	queriesByPanel := queryModels.GroupQueriesByPanelId(dashboard.Data)
 	queries, ok := queriesByPanel[panelId]
 	if !ok {
 		return dtos.MetricRequest{}, ErrPublicDashboardPanelNotFound
@@ -246,7 +247,7 @@ func (pd *PublicDashboardServiceImpl) buildPublicDashboardMetricRequest(ctx cont
 
 // BuildAnonymousUser creates a user with permissions to read from all datasources used in the dashboard
 func (pd *PublicDashboardServiceImpl) BuildAnonymousUser(ctx context.Context, dashboard *models.Dashboard) (*user.SignedInUser, error) {
-	datasourceUids := models.GetUniqueDashboardDatasourceUids(dashboard.Data)
+	datasourceUids := queryModels.GetUniqueDashboardDatasourceUids(dashboard.Data)
 
 	// Create a temp user with read-only datasource permissions
 	anonymousUser := &user.SignedInUser{OrgID: dashboard.OrgId, Permissions: make(map[int64]map[string][]string)}

pkg/services/query/models/dashboard_queries.go

@@ -1,7 +1,8 @@
-package models
+package query
 
 import (
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/expr"
 )
 
 func GetUniqueDashboardDatasourceUids(dashboard *simplejson.Json) []string {
@@ -60,6 +61,17 @@ func GroupQueriesByPanelId(dashboard *simplejson.Json) map[int64][]*simplejson.J
 	return result
 }
 
+func HasExpressionQuery(queries []*simplejson.Json) bool {
+	for _, query := range queries {
+		uid := GetDataSourceUidFromJson(query)
+		if expr.IsDataSource(uid) {
+			return true
+		}
+	}
+
+	return false
+}
+
 func GroupQueriesByDataSource(queries []*simplejson.Json) (result [][]*simplejson.Json) {
 	byDataSource := make(map[string][]*simplejson.Json)
 

pkg/services/query/models/dashboard_queries_test.go

@@ -1,4 +1,4 @@
-package models
+package query
 
 import (
 	"testing"
@@ -87,6 +87,53 @@ const (
   "schemaVersion": 35
 }`
 
+	dashboardWithQueriesAndExpression = `
+{
+  "panels": [
+    {
+      "id": 2,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "_yxMP8Ynk"
+          },
+          "exemplar": true,
+          "expr": "go_goroutines{job=\"$job\"}",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        },
+		{
+          "datasource": {
+            "name": "Expression",
+            "type": "__expr__",
+            "uid": "__expr__"
+          },
+          "expression": "$A + 1",
+          "hide": false,
+          "refId": "EXPRESSION",
+          "type": "math"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "promds2"
+          },
+          "exemplar": true,
+          "expr": "query2",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "B"
+        }
+      ],
+      "title": "Panel Title",
+      "type": "timeseries"
+    }
+  ],
+  "schemaVersion": 35
+}`
+
 	dashboardWithMixedDatasource = `
 {
   "panels": [
@@ -286,6 +333,25 @@ func TestGetUniqueDashboardDatasourceUids(t *testing.T) {
 	})
 }
 
+func TestHasExpressionQuery(t *testing.T) {
+	t.Run("will return true when expression query exists", func(t *testing.T) {
+		json, err := simplejson.NewJson([]byte(dashboardWithQueriesAndExpression))
+		require.NoError(t, err)
+
+		queries := GroupQueriesByPanelId(json)
+		panelId := int64(2)
+		require.True(t, HasExpressionQuery(queries[panelId]))
+	})
+	t.Run("will return false when no expression query exists", func(t *testing.T) {
+		json, err := simplejson.NewJson([]byte(dashboardWithMixedDatasource))
+		require.NoError(t, err)
+
+		queries := GroupQueriesByPanelId(json)
+		panelId := int64(2)
+		require.False(t, HasExpressionQuery(queries[panelId]))
+	})
+}
+
 func TestGroupQueriesByPanelId(t *testing.T) {
 	t.Run("can extract queries from dashboard with panel datasource string that has no datasource on panel targets", func(t *testing.T) {
 		json, err := simplejson.NewJson([]byte(oldStyleDashboard))

pkg/services/query/query.go

@@ -16,6 +16,7 @@ import (
 	"github.com/grafana/grafana/pkg/plugins/adapters"
 	"github.com/grafana/grafana/pkg/services/datasources"
 	"github.com/grafana/grafana/pkg/services/oauthtoken"
+	queryModels "github.com/grafana/grafana/pkg/services/query/models"
 	"github.com/grafana/grafana/pkg/services/user"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb/grafanads"
@@ -80,13 +81,15 @@ func (s *Service) QueryData(ctx context.Context, user *user.SignedInUser, skipCa
 
 // QueryData can process queries and return query responses.
 func (s *Service) QueryDataMultipleSources(ctx context.Context, user *user.SignedInUser, skipCache bool, reqDTO dtos.MetricRequest, handleExpressions bool) (*backend.QueryDataResponse, error) {
-	byDataSource := models.GroupQueriesByDataSource(reqDTO.Queries)
+	byDataSource := queryModels.GroupQueriesByDataSource(reqDTO.Queries)
 
-	if len(byDataSource) == 1 {
+	// The expression service will handle mixed datasources, so we don't need to group them when an expression is present.
+	if queryModels.HasExpressionQuery(reqDTO.Queries) || len(byDataSource) == 1 {
 		return s.QueryData(ctx, user, skipCache, reqDTO, handleExpressions)
 	} else {
 		resp := backend.NewQueryDataResponse()
 
+		// create new reqDTO with only the queries for that datasource
 		for _, queries := range byDataSource {
 			subDTO := reqDTO.CloneWithQueries(queries)
 

pkg/services/query/query_test.go

@@ -2,10 +2,12 @@ package query_test
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"testing"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana/pkg/expr"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/oauth2"
 
@@ -15,6 +17,7 @@ import (
 	"github.com/grafana/grafana/pkg/plugins"
 	acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
 	"github.com/grafana/grafana/pkg/services/datasources"
+	fakeDatasources "github.com/grafana/grafana/pkg/services/datasources/fakes"
 	dsSvc "github.com/grafana/grafana/pkg/services/datasources/service"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/query"
@@ -25,6 +28,92 @@ import (
 	"github.com/grafana/grafana/pkg/services/user"
 )
 
+func TestQueryDataMultipleSources(t *testing.T) {
+	t.Run("can query multiple datasources", func(t *testing.T) {
+		tc := setup(t)
+		query1, err := simplejson.NewJson([]byte(`
+			{
+				"datasource": {
+					"type": "mysql",
+					"uid": "ds1"
+				}
+			}
+		`))
+		require.NoError(t, err)
+		query2, err := simplejson.NewJson([]byte(`
+			{
+				"datasource": {
+					"type": "mysql",
+					"uid": "ds2"
+				}
+			}
+		`))
+		require.NoError(t, err)
+		queries := []*simplejson.Json{query1, query2}
+		reqDTO := dtos.MetricRequest{
+			From:                       "2022-01-01",
+			To:                         "2022-01-02",
+			Queries:                    queries,
+			Debug:                      false,
+			PublicDashboardAccessToken: "abc123",
+			HTTPRequest:                nil,
+		}
+
+		_, err = tc.queryService.QueryDataMultipleSources(context.Background(), nil, true, reqDTO, false)
+
+		require.NoError(t, err)
+	})
+
+	t.Run("can query multiple datasources with an expression present", func(t *testing.T) {
+		tc := setup(t)
+		query1, err := simplejson.NewJson([]byte(`
+			{
+				"datasource": {
+					"type": "mysql",
+					"uid": "ds1"
+				}
+			}
+		`))
+		require.NoError(t, err)
+		query2, err := simplejson.NewJson([]byte(`
+			{
+				"datasource": {
+					"type": "mysql",
+					"uid": "ds2"
+				}
+			}
+		`))
+		require.NoError(t, err)
+		query3, err := simplejson.NewJson([]byte(`
+			{
+				"datasource": {
+					"name": "Expression",
+					"type": "__expr__",
+					"uid": "__expr__"
+				},
+				"expression": "$A + 1",
+				"hide": false,
+				"refId": "EXPRESSION",
+				"type": "math"
+			}
+		`))
+		require.NoError(t, err)
+		queries := []*simplejson.Json{query1, query2, query3}
+		reqDTO := dtos.MetricRequest{
+			From:                       "2022-01-01",
+			To:                         "2022-01-02",
+			Queries:                    queries,
+			Debug:                      false,
+			PublicDashboardAccessToken: "abc123",
+			HTTPRequest:                nil,
+		}
+
+		_, err = tc.queryService.QueryDataMultipleSources(context.Background(), nil, true, reqDTO, false)
+
+		require.NoError(t, err)
+	})
+}
+
 func TestQueryData(t *testing.T) {
 	t.Run("it auth custom headers to the request", func(t *testing.T) {
 		token := &oauth2.Token{
@@ -96,6 +185,11 @@ func setup(t *testing.T) *testContext {
 	ss := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
 	ssvc := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
 	ds := dsSvc.ProvideService(nil, ssvc, ss, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService())
+	fakeDatasourceService := &fakeDatasources.FakeDataSourceService{
+		DataSources:           nil,
+		SimulatePluginFailure: false,
+	}
+	exprService := expr.ProvideService(nil, pc, fakeDatasourceService)
 
 	return &testContext{
 		pluginContext:          pc,
@@ -103,7 +197,7 @@ func setup(t *testing.T) *testContext {
 		dataSourceCache:        dc,
 		oauthTokenService:      tc,
 		pluginRequestValidator: rv,
-		queryService:           query.ProvideService(nil, dc, nil, rv, ds, pc, tc),
+		queryService:           query.ProvideService(nil, dc, exprService, rv, ds, pc, tc),
 	}
 }
 
@@ -167,5 +261,11 @@ type fakePluginClient struct {
 
 func (c *fakePluginClient) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
 	c.req = req
-	return nil, nil
+
+	// If an expression query ends up getting directly queried, we want it to return an error in our test.
+	if req.PluginContext.PluginID == "__expr__" {
+		return nil, errors.New("cant query an expression datasource")
+	}
+
+	return &backend.QueryDataResponse{Responses: make(backend.Responses)}, nil
 }

public/app/features/query/state/runRequest.ts

@@ -174,7 +174,11 @@ export function callQueryMethod(
   request: DataQueryRequest,
   queryFunction?: typeof datasource.query
 ) {
-  // If any query has an expression, use the expression endpoint
+  // If its a public datasource, just return the result. Expressions will be handled on the backend.
+  if (datasource.type === 'public-ds') {
+    return from(datasource.query(request));
+  }
+
   for (const target of request.targets) {
     if (isExpressionReference(target.datasource)) {
       return expressionDatasource.query(request as DataQueryRequest<ExpressionQuery>);