From f4bbce15a0cb328872d7c9e6bcb895e45964291b Mon Sep 17 00:00:00 2001 From: owensmallwood Date: Wed, 31 Aug 2022 09:11:10 -0600 Subject: [PATCH] Public Dashboards: Add Expressions Support (#54336) Adds support for expressions with public dashboards --- .../public_dashboard_service_mock.go | 4 +- .../publicdashboards/service/service.go | 5 +- .../query}/models/dashboard_queries.go | 14 ++- .../query}/models/dashboard_queries_test.go | 68 +++++++++++- pkg/services/query/query.go | 7 +- pkg/services/query/query_test.go | 104 +++++++++++++++++- public/app/features/query/state/runRequest.ts | 6 +- 7 files changed, 197 insertions(+), 11 deletions(-) rename pkg/{ => services/query}/models/dashboard_queries.go (90%) rename pkg/{ => services/query}/models/dashboard_queries_test.go (86%) diff --git a/pkg/services/publicdashboards/public_dashboard_service_mock.go b/pkg/services/publicdashboards/public_dashboard_service_mock.go index 1ba4f77b013..8d683be9748 100644 --- a/pkg/services/publicdashboards/public_dashboard_service_mock.go +++ b/pkg/services/publicdashboards/public_dashboard_service_mock.go @@ -6,10 +6,10 @@ import ( context "context" testing "testing" - mock "github.com/stretchr/testify/mock" + "github.com/grafana/grafana-plugin-sdk-go/backend" models "github.com/grafana/grafana/pkg/models" publicdashboardsmodels "github.com/grafana/grafana/pkg/services/publicdashboards/models" - "github.com/grafana/grafana-plugin-sdk-go/backend" + mock "github.com/stretchr/testify/mock" user "github.com/grafana/grafana/pkg/services/user" ) diff --git a/pkg/services/publicdashboards/service/service.go b/pkg/services/publicdashboards/service/service.go index d878b970bae..dbe1fdac2b8 100644 --- a/pkg/services/publicdashboards/service/service.go +++ b/pkg/services/publicdashboards/service/service.go @@ -16,6 +16,7 @@ import ( . "github.com/grafana/grafana/pkg/services/publicdashboards/models" "github.com/grafana/grafana/pkg/services/publicdashboards/validation" "github.com/grafana/grafana/pkg/services/query" + queryModels "github.com/grafana/grafana/pkg/services/query/models" "github.com/grafana/grafana/pkg/services/user" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/intervalv2" @@ -222,7 +223,7 @@ func (pd *PublicDashboardServiceImpl) GetQueryDataResponse(ctx context.Context, // dashboard and returns a metrics request to be sent to query backend func (pd *PublicDashboardServiceImpl) buildPublicDashboardMetricRequest(ctx context.Context, dashboard *models.Dashboard, publicDashboard *PublicDashboard, panelId int64, reqDTO *PublicDashboardQueryDTO) (dtos.MetricRequest, error) { // group queries by panel - queriesByPanel := models.GroupQueriesByPanelId(dashboard.Data) + queriesByPanel := queryModels.GroupQueriesByPanelId(dashboard.Data) queries, ok := queriesByPanel[panelId] if !ok { return dtos.MetricRequest{}, ErrPublicDashboardPanelNotFound @@ -246,7 +247,7 @@ func (pd *PublicDashboardServiceImpl) buildPublicDashboardMetricRequest(ctx cont // BuildAnonymousUser creates a user with permissions to read from all datasources used in the dashboard func (pd *PublicDashboardServiceImpl) BuildAnonymousUser(ctx context.Context, dashboard *models.Dashboard) (*user.SignedInUser, error) { - datasourceUids := models.GetUniqueDashboardDatasourceUids(dashboard.Data) + datasourceUids := queryModels.GetUniqueDashboardDatasourceUids(dashboard.Data) // Create a temp user with read-only datasource permissions anonymousUser := &user.SignedInUser{OrgID: dashboard.OrgId, Permissions: make(map[int64]map[string][]string)} diff --git a/pkg/models/dashboard_queries.go b/pkg/services/query/models/dashboard_queries.go similarity index 90% rename from pkg/models/dashboard_queries.go rename to pkg/services/query/models/dashboard_queries.go index 5f3338f00a2..92202d8d4b0 100644 --- a/pkg/models/dashboard_queries.go +++ b/pkg/services/query/models/dashboard_queries.go @@ -1,7 +1,8 @@ -package models +package query import ( "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/expr" ) func GetUniqueDashboardDatasourceUids(dashboard *simplejson.Json) []string { @@ -60,6 +61,17 @@ func GroupQueriesByPanelId(dashboard *simplejson.Json) map[int64][]*simplejson.J return result } +func HasExpressionQuery(queries []*simplejson.Json) bool { + for _, query := range queries { + uid := GetDataSourceUidFromJson(query) + if expr.IsDataSource(uid) { + return true + } + } + + return false +} + func GroupQueriesByDataSource(queries []*simplejson.Json) (result [][]*simplejson.Json) { byDataSource := make(map[string][]*simplejson.Json) diff --git a/pkg/models/dashboard_queries_test.go b/pkg/services/query/models/dashboard_queries_test.go similarity index 86% rename from pkg/models/dashboard_queries_test.go rename to pkg/services/query/models/dashboard_queries_test.go index 553c58a6c37..7bf14c62343 100644 --- a/pkg/models/dashboard_queries_test.go +++ b/pkg/services/query/models/dashboard_queries_test.go @@ -1,4 +1,4 @@ -package models +package query import ( "testing" @@ -87,6 +87,53 @@ const ( "schemaVersion": 35 }` + dashboardWithQueriesAndExpression = ` +{ + "panels": [ + { + "id": 2, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_yxMP8Ynk" + }, + "exemplar": true, + "expr": "go_goroutines{job=\"$job\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$A + 1", + "hide": false, + "refId": "EXPRESSION", + "type": "math" + }, + { + "datasource": { + "type": "prometheus", + "uid": "promds2" + }, + "exemplar": true, + "expr": "query2", + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "Panel Title", + "type": "timeseries" + } + ], + "schemaVersion": 35 +}` + dashboardWithMixedDatasource = ` { "panels": [ @@ -286,6 +333,25 @@ func TestGetUniqueDashboardDatasourceUids(t *testing.T) { }) } +func TestHasExpressionQuery(t *testing.T) { + t.Run("will return true when expression query exists", func(t *testing.T) { + json, err := simplejson.NewJson([]byte(dashboardWithQueriesAndExpression)) + require.NoError(t, err) + + queries := GroupQueriesByPanelId(json) + panelId := int64(2) + require.True(t, HasExpressionQuery(queries[panelId])) + }) + t.Run("will return false when no expression query exists", func(t *testing.T) { + json, err := simplejson.NewJson([]byte(dashboardWithMixedDatasource)) + require.NoError(t, err) + + queries := GroupQueriesByPanelId(json) + panelId := int64(2) + require.False(t, HasExpressionQuery(queries[panelId])) + }) +} + func TestGroupQueriesByPanelId(t *testing.T) { t.Run("can extract queries from dashboard with panel datasource string that has no datasource on panel targets", func(t *testing.T) { json, err := simplejson.NewJson([]byte(oldStyleDashboard)) diff --git a/pkg/services/query/query.go b/pkg/services/query/query.go index 52d51f7eb7a..fb14d554697 100644 --- a/pkg/services/query/query.go +++ b/pkg/services/query/query.go @@ -16,6 +16,7 @@ import ( "github.com/grafana/grafana/pkg/plugins/adapters" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/oauthtoken" + queryModels "github.com/grafana/grafana/pkg/services/query/models" "github.com/grafana/grafana/pkg/services/user" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/grafanads" @@ -80,13 +81,15 @@ func (s *Service) QueryData(ctx context.Context, user *user.SignedInUser, skipCa // QueryData can process queries and return query responses. func (s *Service) QueryDataMultipleSources(ctx context.Context, user *user.SignedInUser, skipCache bool, reqDTO dtos.MetricRequest, handleExpressions bool) (*backend.QueryDataResponse, error) { - byDataSource := models.GroupQueriesByDataSource(reqDTO.Queries) + byDataSource := queryModels.GroupQueriesByDataSource(reqDTO.Queries) - if len(byDataSource) == 1 { + // The expression service will handle mixed datasources, so we don't need to group them when an expression is present. + if queryModels.HasExpressionQuery(reqDTO.Queries) || len(byDataSource) == 1 { return s.QueryData(ctx, user, skipCache, reqDTO, handleExpressions) } else { resp := backend.NewQueryDataResponse() + // create new reqDTO with only the queries for that datasource for _, queries := range byDataSource { subDTO := reqDTO.CloneWithQueries(queries) diff --git a/pkg/services/query/query_test.go b/pkg/services/query/query_test.go index bcb0105674f..33c8fc1f6bf 100644 --- a/pkg/services/query/query_test.go +++ b/pkg/services/query/query_test.go @@ -2,10 +2,12 @@ package query_test import ( "context" + "errors" "net/http" "testing" "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/expr" "github.com/stretchr/testify/require" "golang.org/x/oauth2" @@ -15,6 +17,7 @@ import ( "github.com/grafana/grafana/pkg/plugins" acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/datasources" + fakeDatasources "github.com/grafana/grafana/pkg/services/datasources/fakes" dsSvc "github.com/grafana/grafana/pkg/services/datasources/service" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/query" @@ -25,6 +28,92 @@ import ( "github.com/grafana/grafana/pkg/services/user" ) +func TestQueryDataMultipleSources(t *testing.T) { + t.Run("can query multiple datasources", func(t *testing.T) { + tc := setup(t) + query1, err := simplejson.NewJson([]byte(` + { + "datasource": { + "type": "mysql", + "uid": "ds1" + } + } + `)) + require.NoError(t, err) + query2, err := simplejson.NewJson([]byte(` + { + "datasource": { + "type": "mysql", + "uid": "ds2" + } + } + `)) + require.NoError(t, err) + queries := []*simplejson.Json{query1, query2} + reqDTO := dtos.MetricRequest{ + From: "2022-01-01", + To: "2022-01-02", + Queries: queries, + Debug: false, + PublicDashboardAccessToken: "abc123", + HTTPRequest: nil, + } + + _, err = tc.queryService.QueryDataMultipleSources(context.Background(), nil, true, reqDTO, false) + + require.NoError(t, err) + }) + + t.Run("can query multiple datasources with an expression present", func(t *testing.T) { + tc := setup(t) + query1, err := simplejson.NewJson([]byte(` + { + "datasource": { + "type": "mysql", + "uid": "ds1" + } + } + `)) + require.NoError(t, err) + query2, err := simplejson.NewJson([]byte(` + { + "datasource": { + "type": "mysql", + "uid": "ds2" + } + } + `)) + require.NoError(t, err) + query3, err := simplejson.NewJson([]byte(` + { + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$A + 1", + "hide": false, + "refId": "EXPRESSION", + "type": "math" + } + `)) + require.NoError(t, err) + queries := []*simplejson.Json{query1, query2, query3} + reqDTO := dtos.MetricRequest{ + From: "2022-01-01", + To: "2022-01-02", + Queries: queries, + Debug: false, + PublicDashboardAccessToken: "abc123", + HTTPRequest: nil, + } + + _, err = tc.queryService.QueryDataMultipleSources(context.Background(), nil, true, reqDTO, false) + + require.NoError(t, err) + }) +} + func TestQueryData(t *testing.T) { t.Run("it auth custom headers to the request", func(t *testing.T) { token := &oauth2.Token{ @@ -96,6 +185,11 @@ func setup(t *testing.T) *testContext { ss := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger")) ssvc := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore()) ds := dsSvc.ProvideService(nil, ssvc, ss, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService()) + fakeDatasourceService := &fakeDatasources.FakeDataSourceService{ + DataSources: nil, + SimulatePluginFailure: false, + } + exprService := expr.ProvideService(nil, pc, fakeDatasourceService) return &testContext{ pluginContext: pc, @@ -103,7 +197,7 @@ func setup(t *testing.T) *testContext { dataSourceCache: dc, oauthTokenService: tc, pluginRequestValidator: rv, - queryService: query.ProvideService(nil, dc, nil, rv, ds, pc, tc), + queryService: query.ProvideService(nil, dc, exprService, rv, ds, pc, tc), } } @@ -167,5 +261,11 @@ type fakePluginClient struct { func (c *fakePluginClient) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { c.req = req - return nil, nil + + // If an expression query ends up getting directly queried, we want it to return an error in our test. + if req.PluginContext.PluginID == "__expr__" { + return nil, errors.New("cant query an expression datasource") + } + + return &backend.QueryDataResponse{Responses: make(backend.Responses)}, nil } diff --git a/public/app/features/query/state/runRequest.ts b/public/app/features/query/state/runRequest.ts index 768262916f7..9f0f0490707 100644 --- a/public/app/features/query/state/runRequest.ts +++ b/public/app/features/query/state/runRequest.ts @@ -174,7 +174,11 @@ export function callQueryMethod( request: DataQueryRequest, queryFunction?: typeof datasource.query ) { - // If any query has an expression, use the expression endpoint + // If its a public datasource, just return the result. Expressions will be handled on the backend. + if (datasource.type === 'public-ds') { + return from(datasource.query(request)); + } + for (const target of request.targets) { if (isExpressionReference(target.datasource)) { return expressionDatasource.query(request as DataQueryRequest);