Search: support datasource template variables when parsing dashboard JSON models (#51587)

* init

* support template variables

* support variables without curly braces

* add todo for `__all` case

* fix `$__all` case for non-multivalue

* extract some functions

* fix flakinesss

* support `$__all` and `default` template variables

* add todo

* compilation fix

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

pkg/services/export/export_dashboards.go

@@ -11,11 +11,12 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/grafana/grafana/pkg/infra/filestorage"
+	"github.com/grafana/grafana/pkg/services/searchV2"
 	"github.com/grafana/grafana/pkg/services/searchV2/extract"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 )
 
-func exportDashboards(helper *commitHelper, job *gitExportJob, lookup dsLookup) error {
+func exportDashboards(helper *commitHelper, job *gitExportJob) error {
 	alias := make(map[string]string, 100)
 	ids := make(map[int64]string, 100)
 	folders := make(map[int64]string, 100)
@@ -25,13 +26,18 @@ func exportDashboards(helper *commitHelper, job *gitExportJob, lookup dsLookup)
 		folders[0] = job.cfg.GeneralFolderPath // "general"
 	}
 
+	lookup, err := searchV2.LoadDatasourceLookup(helper.ctx, helper.orgID, job.sql)
+	if err != nil {
+		return err
+	}
+
 	rootDir := path.Join(helper.orgDir, "root")
 	folderStructure := commitOptions{
 		when:    time.Now(),
 		comment: "Exported folder structure",
 	}
 
-	err := job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
+	err = job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
 		type dashDataQueryResult struct {
 			Id       int64
 			UID      string `xorm:"uid"`

pkg/services/export/export_ds.go

@@ -6,41 +6,26 @@ import (
 	"sort"
 
 	"github.com/grafana/grafana/pkg/services/datasources"
-	"github.com/grafana/grafana/pkg/services/searchV2/extract"
 )
 
-type dsLookup func(ref *extract.DataSourceRef) *extract.DataSourceRef
+func exportDataSources(helper *commitHelper, job *gitExportJob) error {
-
-func exportDataSources(helper *commitHelper, job *gitExportJob, save bool) (dsLookup, error) {
 	cmd := &datasources.GetDataSourcesQuery{
 		OrgId: job.orgID,
 	}
 	err := job.sql.GetDataSources(helper.ctx, cmd)
 	if err != nil {
-		return nil, err
+		return nil
 	}
 
 	sort.SliceStable(cmd.Result, func(i, j int) bool {
 		return cmd.Result[i].Created.After(cmd.Result[j].Created)
 	})
 
-	byUID := make(map[string]*extract.DataSourceRef, len(cmd.Result))
-	byName := make(map[string]*extract.DataSourceRef, len(cmd.Result))
 	for _, ds := range cmd.Result {
-		ref := &extract.DataSourceRef{
-			UID:  ds.Uid,
-			Type: ds.Type,
-		}
-		byUID[ds.Uid] = ref
-		byName[ds.Name] = ref
-		if !save {
-			continue
-		}
-
 		ds.OrgId = 0
 		ds.Version = 0
 		ds.SecureJsonData = map[string][]byte{
-			"TODO": []byte("secret store lookup"),
+			"TODO": []byte("XXX"),
 		}
 
 		err := helper.add(commitOptions{
@@ -54,26 +39,9 @@ func exportDataSources(helper *commitHelper, job *gitExportJob, save bool) (dsLo
 			comment: fmt.Sprintf("Add datasource: %s", ds.Name),
 		})
 		if err != nil {
-			return nil, err
+			return err
 		}
 	}
 
-	// Return the lookup function
+	return nil
-	return func(ref *extract.DataSourceRef) *extract.DataSourceRef {
-		if ref == nil || ref.UID == "" {
-			return &extract.DataSourceRef{
-				UID:  "default.uid",
-				Type: "default.type",
-			}
-		}
-		v, ok := byUID[ref.UID]
-		if ok {
-			return v
-		}
-		v, ok = byName[ref.UID]
-		if ok {
-			return v
-		}
-		return nil
-	}, nil
 }

pkg/services/export/git_export_job.go

@@ -172,19 +172,16 @@ func (e *gitExportJob) doExportWithHistory() error {
 
 func (e *gitExportJob) doOrgExportWithHistory(helper *commitHelper) error {
 	include := e.cfg.Include
-	lookup, err := exportDataSources(helper, e, include.DS)
-	if err != nil {
-		return err
-	}
-
-	if include.Dash {
-		err = exportDashboards(helper, e, lookup)
-		if err != nil {
-			return err
-		}
-	}
 
 	exporters := []simpleExporter{}
+	if include.Dash {
+		exporters = append(exporters, exportDashboards)
+	}
+
+	if include.DS {
+		exporters = append(exporters, exportDataSources)
+	}
+
 	if include.Auth {
 		exporters = append(exporters, dumpAuthTables)
 	}
@@ -207,12 +204,12 @@ func (e *gitExportJob) doOrgExportWithHistory(helper *commitHelper) error {
 	}
 
 	for _, fn := range exporters {
-		err = fn(helper, e)
+		err := fn(helper, e)
 		if err != nil {
 			return err
 		}
 	}
-	return err
+	return nil
 }
 
 /**

pkg/services/searchV2/extract/dashboard.go

@@ -3,6 +3,7 @@ package extract
 import (
 	"io"
 	"strconv"
+	"strings"
 
 	jsoniter "github.com/json-iterator/go"
 )
@@ -11,6 +12,99 @@ func logf(format string, a ...interface{}) {
 	//fmt.Printf(format, a...)
 }
 
+type templateVariable struct {
+	current struct {
+		value interface{}
+	}
+	name         string
+	query        interface{}
+	variableType string
+}
+
+type datasourceVariableLookup struct {
+	variableNameToRefs map[string][]DataSourceRef
+	dsLookup           DatasourceLookup
+}
+
+func (d *datasourceVariableLookup) getDsRefsByTemplateVariableValue(value string, datasourceType string) []DataSourceRef {
+	switch value {
+	case "default":
+		// can be the default DS, or a DS with UID="default"
+		candidateDs := d.dsLookup.ByRef(&DataSourceRef{UID: value})
+		if candidateDs == nil {
+			// get the actual default DS
+			candidateDs = d.dsLookup.ByRef(nil)
+		}
+		return []DataSourceRef{*candidateDs}
+	case "$__all":
+		// TODO: filter datasources by template variable's regex
+		return d.dsLookup.ByType(datasourceType)
+	case "":
+		return []DataSourceRef{}
+	case "No data sources found":
+		return []DataSourceRef{}
+	default:
+		return []DataSourceRef{
+			{
+				UID:  value,
+				Type: datasourceType,
+			},
+		}
+	}
+}
+
+func (d *datasourceVariableLookup) add(templateVariable templateVariable) {
+	var refs []DataSourceRef
+
+	datasourceType, isDataSourceTypeValid := templateVariable.query.(string)
+	if !isDataSourceTypeValid {
+		d.variableNameToRefs[templateVariable.name] = refs
+		return
+	}
+
+	if values, multiValueVariable := templateVariable.current.value.([]interface{}); multiValueVariable {
+		for _, value := range values {
+			if valueAsString, ok := value.(string); ok {
+				refs = append(refs, d.getDsRefsByTemplateVariableValue(valueAsString, datasourceType)...)
+			}
+		}
+	}
+
+	if value, stringValue := templateVariable.current.value.(string); stringValue {
+		refs = append(refs, d.getDsRefsByTemplateVariableValue(value, datasourceType)...)
+	}
+
+	d.variableNameToRefs[templateVariable.name] = unique(refs)
+}
+
+func unique(refs []DataSourceRef) []DataSourceRef {
+	var uniqueRefs []DataSourceRef
+	uidPresence := make(map[string]bool)
+	for _, ref := range refs {
+		if !uidPresence[ref.UID] {
+			uidPresence[ref.UID] = true
+			uniqueRefs = append(uniqueRefs, ref)
+		}
+	}
+	return uniqueRefs
+}
+
+func (d *datasourceVariableLookup) getDatasourceRefs(name string) []DataSourceRef {
+	refs, ok := d.variableNameToRefs[name]
+	if ok {
+		return refs
+	}
+
+	return []DataSourceRef{}
+}
+
+func newDatasourceVariableLookup(dsLookup DatasourceLookup) *datasourceVariableLookup {
+	return &datasourceVariableLookup{
+		variableNameToRefs: make(map[string][]DataSourceRef),
+		dsLookup:           dsLookup,
+	}
+}
+
 // nolint:gocyclo
 // ReadDashboard will take a byte stream and return dashboard info
 func ReadDashboard(stream io.Reader, lookup DatasourceLookup) (*DashboardInfo, error) {
@@ -18,6 +112,8 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) (*DashboardInfo, e
 
 	iter := jsoniter.Parse(jsoniter.ConfigDefault, stream, 1024)
 
+	datasourceVariablesLookup := newDatasourceVariableLookup(lookup)
+
 	for l1Field := iter.ReadObject(); l1Field != ""; l1Field = iter.ReadObject() {
 		// Skip null values so we don't need special int handling
 		if iter.WhatIsNext() == jsoniter.NilValue {
@@ -112,13 +208,34 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) (*DashboardInfo, e
 			for sub := iter.ReadObject(); sub != ""; sub = iter.ReadObject() {
 				if sub == "list" {
 					for iter.ReadArray() {
+						templateVariable := templateVariable{}
+
 						for k := iter.ReadObject(); k != ""; k = iter.ReadObject() {
-							if k == "name" {
+							switch k {
-								dash.TemplateVars = append(dash.TemplateVars, iter.ReadString())
+							case "name":
-							} else {
+								name := iter.ReadString()
+								dash.TemplateVars = append(dash.TemplateVars, name)
+								templateVariable.name = name
+							case "type":
+								templateVariable.variableType = iter.ReadString()
+							case "query":
+								templateVariable.query = iter.Read()
+							case "current":
+								for c := iter.ReadObject(); c != ""; c = iter.ReadObject() {
+									if c == "value" {
+										templateVariable.current.value = iter.Read()
+									} else {
+										iter.Skip()
+									}
+								}
+							default:
 								iter.Skip()
 							}
 						}
+
+						if templateVariable.variableType == "datasource" {
+							datasourceVariablesLookup.add(templateVariable)
+						}
 					}
 				} else {
 					iter.Skip()
@@ -139,6 +256,8 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) (*DashboardInfo, e
 		}
 	}
 
+	replaceDatasourceVariables(dash, datasourceVariablesLookup)
+
 	targets := newTargetInfo(lookup)
 	for _, panel := range dash.Panels {
 		targets.addPanel(panel)
@@ -148,6 +267,43 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) (*DashboardInfo, e
 	return dash, iter.Error
 }
 
+func replaceDatasourceVariables(dash *DashboardInfo, datasourceVariablesLookup *datasourceVariableLookup) {
+	for i, panel := range dash.Panels {
+		var dsVariableRefs []DataSourceRef
+		var dsRefs []DataSourceRef
+
+		// partition into actual datasource references and variables
+		for i := range panel.Datasource {
+			isVariableRef := strings.HasPrefix(panel.Datasource[i].UID, "$")
+			if isVariableRef {
+				dsVariableRefs = append(dsVariableRefs, panel.Datasource[i])
+			} else {
+				dsRefs = append(dsRefs, panel.Datasource[i])
+			}
+		}
+
+		dash.Panels[i].Datasource = append(dsRefs, findDatasourceRefsForVariables(dsVariableRefs, datasourceVariablesLookup)...)
+	}
+}
+
+func getDataSourceVariableName(dsVariableRef DataSourceRef) string {
+	if strings.HasPrefix(dsVariableRef.UID, "${") {
+		return strings.TrimPrefix(strings.TrimSuffix(dsVariableRef.UID, "}"), "${")
+	}
+
+	return strings.TrimPrefix(dsVariableRef.UID, "$")
+}
+
+func findDatasourceRefsForVariables(dsVariableRefs []DataSourceRef, datasourceVariablesLookup *datasourceVariableLookup) []DataSourceRef {
+	var referencedDs []DataSourceRef
+	for _, dsVariableRef := range dsVariableRefs {
+		variableName := getDataSourceVariableName(dsVariableRef)
+		refs := datasourceVariablesLookup.getDatasourceRefs(variableName)
+		referencedDs = append(referencedDs, refs...)
+	}
+	return referencedDs
+}
+
 // will always return strings for now
 func readPanelInfo(iter *jsoniter.Iterator, lookup DatasourceLookup) PanelInfo {
 	panel := PanelInfo{}

pkg/services/searchV2/extract/dashboard_test.go

@@ -4,28 +4,61 @@ import (
 	"encoding/json"
 	"os"
 	"path/filepath"
+	"sort"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+type dsLookup struct {
+}
+
+func (d *dsLookup) ByRef(ref *DataSourceRef) *DataSourceRef {
+	if ref == nil || ref.UID == "" {
+		return &DataSourceRef{
+			UID:  "default.uid",
+			Type: "default.type",
+		}
+	}
+
+	if ref.UID == "default" {
+		return nil
+	}
+	return ref
+}
+
+func (d *dsLookup) ByType(dsType string) []DataSourceRef {
+	if dsType == "sqlite-datasource" {
+		return []DataSourceRef{
+			{
+				UID:  "sqlite-1",
+				Type: "sqlite-datasource",
+			},
+			{
+				UID:  "sqlite-2",
+				Type: "sqlite-datasource",
+			},
+		}
+	}
+	return make([]DataSourceRef, 0)
+}
+
 func TestReadDashboard(t *testing.T) {
 	inputs := []string{
 		"check-string-datasource-id",
 		"all-panels",
 		"panel-graph/graph-shared-tooltips",
-	}
+		"datasource-variable",
-
+		"default-datasource-variable",
-	// key will allow name or uid
+		"empty-datasource-variable",
-	ds := func(ref *DataSourceRef) *DataSourceRef {
+		"repeated-datasource-variables",
-		if ref == nil || ref.UID == "" {
+		"string-datasource-variable",
-			return &DataSourceRef{
+		"datasource-variable-no-curly-braces",
-				UID:  "default.uid",
+		"all-selected-multi-datasource-variable",
-				Type: "default.type",
+		"all-selected-single-datasource-variable",
-			}
+		"repeated-datasource-variables-with-default",
-		}
-		return ref
 	}
 
 	devdash := "../../../../devenv/dev-dashboards/"
@@ -44,7 +77,9 @@ func TestReadDashboard(t *testing.T) {
 		}
 		require.NoError(t, err)
 
-		dash, err := ReadDashboard(f, ds)
+		dash, err := ReadDashboard(f, &dsLookup{})
+		sortDatasources(dash)
+
 		require.NoError(t, err)
 		out, err := json.MarshalIndent(dash, "", "  ")
 		require.NoError(t, err)
@@ -64,3 +99,16 @@ func TestReadDashboard(t *testing.T) {
 		}
 	}
 }
+
+// assure consistent ordering of datasources to prevent random failures of `assert.JSONEq`
+func sortDatasources(dash *DashboardInfo) {
+	sort.Slice(dash.Datasource, func(i, j int) bool {
+		return strings.Compare(dash.Datasource[i].UID, dash.Datasource[j].UID) > 0
+	})
+
+	for panelId := range dash.Panels {
+		sort.Slice(dash.Panels[panelId].Datasource, func(i, j int) bool {
+			return strings.Compare(dash.Panels[panelId].Datasource[i].UID, dash.Panels[panelId].Datasource[j].UID) > 0
+		})
+	}
+}

pkg/services/searchV2/extract/targets.go

@@ -31,17 +31,17 @@ func (s *targetInfo) addDatasource(iter *jsoniter.Iterator) {
 	switch iter.WhatIsNext() {
 	case jsoniter.StringValue:
 		key := iter.ReadString()
-		ds := s.lookup(&DataSourceRef{UID: key})
+		ds := s.lookup.ByRef(&DataSourceRef{UID: key})
 		s.addRef(ds)
 
 	case jsoniter.NilValue:
-		s.addRef(s.lookup(nil))
+		s.addRef(s.lookup.ByRef(nil))
 		iter.Skip()
 
 	case jsoniter.ObjectValue:
 		ref := &DataSourceRef{}
 		iter.ReadVal(ref)
-		ds := s.lookup(ref)
+		ds := s.lookup.ByRef(ref)
 		s.addRef(ds)
 
 	default:

pkg/services/searchV2/extract/testdata/all-selected-multi-datasource-variable-info.json

@@ -0,0 +1,61 @@
+{
+  "id": 209,
+  "title": "ds-variables",
+  "tags": null,
+  "templateVars": [
+    "sqllite"
+  ],
+  "datasource": [
+    {
+      "uid": "sqlite-2",
+      "type": "sqlite-datasource"
+    },
+    {
+      "uid": "sqlite-1",
+      "type": "sqlite-datasource"
+    }
+  ],
+  "panels": [
+    {
+      "id": 7,
+      "title": "Panel Title",
+      "type": "timeseries",
+      "datasource": [
+        {
+          "uid": "sqlite-2",
+          "type": "sqlite-datasource"
+        },
+        {
+          "uid": "sqlite-1",
+          "type": "sqlite-datasource"
+        }
+      ]
+    },
+    {
+      "id": 3,
+      "title": "Row title",
+      "type": "row"
+    },
+    {
+      "id": 1,
+      "title": "usersss!",
+      "type": "table",
+      "pluginVersion": "9.1.0-pre",
+      "datasource": [
+        {
+          "uid": "sqlite-2",
+          "type": "sqlite-datasource"
+        },
+        {
+          "uid": "sqlite-1",
+          "type": "sqlite-datasource"
+        }
+      ]
+    }
+  ],
+  "schemaVersion": 36,
+  "linkCount": 0,
+  "timeFrom": "now-6h",
+  "timeTo": "now",
+  "timezone": ""
+}

pkg/services/searchV2/extract/testdata/all-selected-multi-datasource-variable.json

@@ -0,0 +1,230 @@
+{
+  "annotations": {
+    "list": [
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 209,
+  "iteration": 1656533909544,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "sqlite-datasource",
+        "uid": "${sqllite}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "repeat": "sqllite",
+      "repeatDirection": "h",
+      "targets": [
+        {
+          "datasource": {
+            "type": "sqlite-datasource",
+            "uid": "${sqllite}"
+          },
+          "queryText": "\n    SELECT CAST(strftime('%s', 'now', '-1 minute') as INTEGER) as time, 4 as value\n    WHERE time >= 1234 and time < 134567\n  ",
+          "queryType": "table",
+          "rawQueryText": "SELECT CAST(strftime('%s', 'now', '-1 minute') as INTEGER) as time, 4 as value \nWHERE time >= $__from / 1000 and time < $__to / 1000",
+          "refId": "A",
+          "timeColumns": [
+            "time",
+            "ts"
+          ]
+        }
+      ],
+      "title": "Panel Title",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "panels": [],
+      "repeat": "sqllite",
+      "repeatDirection": "h",
+      "title": "Row title",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "sqlite-datasource",
+        "uid": "${sqllite}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto",
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 8,
+        "y": 9
+      },
+      "id": 1,
+      "options": {
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "9.1.0-pre",
+      "targets": [
+        {
+          "datasource": {
+            "type": "sqlite-datasource",
+            "uid": "${sqllite}"
+          },
+          "queryText": "select * from user",
+          "queryType": "table",
+          "rawQueryText": "select * from user",
+          "refId": "A",
+          "timeColumns": [
+            "time",
+            "ts"
+          ]
+        }
+      ],
+      "title": "usersss!",
+      "type": "table"
+    }
+  ],
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "allValue": "",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "hide": 0,
+        "includeAll": true,
+        "multi": true,
+        "name": "sqllite",
+        "options": [],
+        "query": "sqlite-datasource",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "ds-variables",
+  "uid": "7BzWolqnk",
+  "version": 36,
+  "weekStart": ""
+}

pkg/services/searchV2/extract/testdata/all-selected-single-datasource-variable-info.json

@@ -0,0 +1,41 @@
+{
+  "id": 208,
+  "title": "new-dashboard-var-ds-test",
+  "tags": null,
+  "templateVars": [
+    "dsVariable"
+  ],
+  "datasource": [
+    {
+      "uid": "sqlite-2",
+      "type": "sqlite-datasource"
+    },
+    {
+      "uid": "sqlite-1",
+      "type": "sqlite-datasource"
+    }
+  ],
+  "panels": [
+    {
+      "id": 2,
+      "title": "Panel Title",
+      "type": "table",
+      "pluginVersion": "9.1.0-pre",
+      "datasource": [
+        {
+          "uid": "sqlite-2",
+          "type": "sqlite-datasource"
+        },
+        {
+          "uid": "sqlite-1",
+          "type": "sqlite-datasource"
+        }
+      ]
+    }
+  ],
+  "schemaVersion": 36,
+  "linkCount": 0,
+  "timeFrom": "now-6h",
+  "timeTo": "now",
+  "timezone": ""
+}

pkg/services/searchV2/extract/testdata/all-selected-single-datasource-variable.json

@@ -0,0 +1,133 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 208,
+  "iteration": 1657048248371,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "sqlite-datasource",
+        "uid": "${dsVariable}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto",
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 16,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "9.1.0-pre",
+      "repeat": "dsVariable",
+      "repeatDirection": "h",
+      "targets": [
+        {
+          "datasource": {
+            "type": "sqlite-datasource",
+            "uid": "${dsVariable}"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Panel Title",
+      "type": "table"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "hide": 0,
+        "includeAll": true,
+        "multi": false,
+        "name": "dsVariable",
+        "options": [],
+        "query": "sqlite-datasource",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "new-dashboard-var-ds-test",
+  "uid": "2HLX_B3nk",
+  "version": 21,
+  "weekStart": ""
+}

pkg/services/searchV2/extract/testdata/datasource-variable-info.json

@@ -0,0 +1,33 @@
+{
+  "id": 209,
+  "title": "ds-variables",
+  "tags": null,
+  "templateVars": [
+    "sqllite"
+  ],
+  "datasource": [
+    {
+      "uid": "SQLite Grafana",
+      "type": "sqlite-datasource"
+    }
+  ],
+  "panels": [
+    {
+      "id": 1,
+      "title": "usersss!",
+      "type": "table",
+      "pluginVersion": "9.1.0-pre",
+      "datasource": [
+        {
+          "uid": "SQLite Grafana",
+          "type": "sqlite-datasource"
+        }
+      ]
+    }
+  ],
+  "schemaVersion": 36,
+  "linkCount": 0,
+  "timeFrom": "now-6h",
+  "timeTo": "now",
+  "timezone": ""
+}

pkg/services/searchV2/extract/testdata/datasource-variable-no-curly-braces-info.json

@@ -0,0 +1,33 @@
+{
+  "id": 209,
+  "title": "ds-variables",
+  "tags": null,
+  "templateVars": [
+    "sqllite"
+  ],
+  "datasource": [
+    {
+      "uid": "SQLite Grafana",
+      "type": "sqlite-datasource"
+    }
+  ],
+  "panels": [
+    {
+      "id": 1,
+      "title": "usersss!",
+      "type": "table",
+      "pluginVersion": "9.1.0-pre",
+      "datasource": [
+        {
+          "uid": "SQLite Grafana",
+          "type": "sqlite-datasource"
+        }
+      ]
+    }
+  ],
+  "schemaVersion": 36,
+  "linkCount": 0,
+  "timeFrom": "now-6h",
+  "timeTo": "now",
+  "timezone": ""
+}

pkg/services/searchV2/extract/testdata/datasource-variable-no-curly-braces.json

@@ -0,0 +1,134 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 209,
+  "iteration": 1656508852566,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "sqlite-datasource",
+        "uid": "$sqllite"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto",
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "9.1.0-pre",
+      "targets": [
+        {
+          "datasource": {
+            "type": "sqlite-datasource",
+            "uid": "$sqllite"
+          },
+          "queryText": "select * from user",
+          "queryType": "table",
+          "rawQueryText": "select * from user",
+          "refId": "A",
+          "timeColumns": [
+            "time",
+            "ts"
+          ]
+        }
+      ],
+      "title": "usersss!",
+      "type": "table"
+    }
+  ],
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "SQLite Grafana",
+          "value": "SQLite Grafana"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "name": "sqllite",
+        "options": [],
+        "query": "sqlite-datasource",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "ds-variables",
+  "uid": "7BzWolqnk",
+  "version": 13,
+  "weekStart": ""
+}

pkg/services/searchV2/extract/testdata/datasource-variable.json

@@ -0,0 +1,134 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 209,
+  "iteration": 1656508852566,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "sqlite-datasource",
+        "uid": "${sqllite}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto",
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "9.1.0-pre",
+      "targets": [
+        {
+          "datasource": {
+            "type": "sqlite-datasource",
+            "uid": "${sqllite}"
+          },
+          "queryText": "select * from user",
+          "queryType": "table",
+          "rawQueryText": "select * from user",
+          "refId": "A",
+          "timeColumns": [
+            "time",
+            "ts"
+          ]
+        }
+      ],
+      "title": "usersss!",
+      "type": "table"
+    }
+  ],
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "SQLite Grafana",
+          "value": "SQLite Grafana"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "name": "sqllite",
+        "options": [],
+        "query": "sqlite-datasource",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "ds-variables",
+  "uid": "7BzWolqnk",
+  "version": 13,
+  "weekStart": ""
+}

pkg/services/searchV2/extract/testdata/default-datasource-variable-info.json

@@ -0,0 +1,33 @@
+{
+  "id": 208,
+  "title": "new-dashboard-var-ds-test",
+  "tags": null,
+  "templateVars": [
+    "dsVariable"
+  ],
+  "datasource": [
+    {
+      "uid": "default.uid",
+      "type": "default.type"
+    }
+  ],
+  "panels": [
+    {
+      "id": 2,
+      "title": "Panel Title",
+      "type": "table",
+      "pluginVersion": "9.1.0-pre",
+      "datasource": [
+        {
+          "uid": "default.uid",
+          "type": "default.type"
+        }
+      ]
+    }
+  ],
+  "schemaVersion": 36,
+  "linkCount": 0,
+  "timeFrom": "now-6h",
+  "timeTo": "now",
+  "timezone": ""
+}

pkg/services/searchV2/extract/testdata/default-datasource-variable.json

@@ -0,0 +1,133 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 208,
+  "iteration": 1657048248373,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "testdata",
+        "uid": "${dsVariable}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto",
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 16,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "9.1.0-pre",
+      "repeat": "dsVariable",
+      "repeatDirection": "h",
+      "targets": [
+        {
+          "datasource": {
+            "type": "testdata",
+            "uid": "${dsVariable}"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Panel Title",
+      "type": "table"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": "default",
+          "value": "default"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "name": "dsVariable",
+        "options": [],
+        "query": "testdata",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "new-dashboard-var-ds-test",
+  "uid": "2HLX_B3nk",
+  "version": 22,
+  "weekStart": ""
+}

pkg/services/searchV2/extract/testdata/empty-datasource-variable-info.json

@@ -0,0 +1,21 @@
+{
+  "id": 209,
+  "title": "ds-variables",
+  "tags": null,
+  "templateVars": [
+    "sqllite"
+  ],
+  "panels": [
+    {
+      "id": 1,
+      "title": "usersss!",
+      "type": "table",
+      "pluginVersion": "9.1.0-pre"
+    }
+  ],
+  "schemaVersion": 36,
+  "linkCount": 0,
+  "timeFrom": "now-6h",
+  "timeTo": "now",
+  "timezone": ""
+}

pkg/services/searchV2/extract/testdata/empty-datasource-variable.json

@@ -0,0 +1,134 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 209,
+  "iteration": 1656513278471,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "sqlite-datasource",
+        "uid": "${sqllite}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto",
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "9.1.0-pre",
+      "targets": [
+        {
+          "datasource": {
+            "type": "sqlite-datasource",
+            "uid": "${sqllite}"
+          },
+          "queryText": "select * from user",
+          "queryType": "table",
+          "rawQueryText": "select * from user",
+          "refId": "A",
+          "timeColumns": [
+            "time",
+            "ts"
+          ]
+        }
+      ],
+      "title": "usersss!",
+      "type": "table"
+    }
+  ],
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "No data sources found",
+          "value": ""
+        },
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "name": "sqllite",
+        "options": [],
+        "query": "sqlite-datasource",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "asdgasd",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "ds-variables",
+  "uid": "7BzWolqnk",
+  "version": 14,
+  "weekStart": ""
+}

pkg/services/searchV2/extract/testdata/repeated-datasource-variables-info.json

@@ -0,0 +1,41 @@
+{
+  "id": 209,
+  "title": "ds-variables",
+  "tags": null,
+  "templateVars": [
+    "sqllite"
+  ],
+  "datasource": [
+    {
+      "uid": "SQLite Grafana2",
+      "type": "sqlite-datasource"
+    },
+    {
+      "uid": "SQLite Grafana",
+      "type": "sqlite-datasource"
+    }
+  ],
+  "panels": [
+    {
+      "id": 1,
+      "title": "usersss!",
+      "type": "table",
+      "pluginVersion": "9.1.0-pre",
+      "datasource": [
+        {
+          "uid": "SQLite Grafana2",
+          "type": "sqlite-datasource"
+        },
+        {
+          "uid": "SQLite Grafana",
+          "type": "sqlite-datasource"
+        }
+      ]
+    }
+  ],
+  "schemaVersion": 36,
+  "linkCount": 0,
+  "timeFrom": "now-6h",
+  "timeTo": "now",
+  "timezone": ""
+}

pkg/services/searchV2/extract/testdata/repeated-datasource-variables-with-default-info.json

@@ -0,0 +1,41 @@
+{
+  "id": 208,
+  "title": "new-dashboard-var-ds-test",
+  "tags": null,
+  "templateVars": [
+    "dsVariable"
+  ],
+  "datasource": [
+    {
+      "uid": "gdev-testdata",
+      "type": "testdata"
+    },
+    {
+      "uid": "default.uid",
+      "type": "default.type"
+    }
+  ],
+  "panels": [
+    {
+      "id": 2,
+      "title": "Panel Title",
+      "type": "table",
+      "pluginVersion": "9.1.0-pre",
+      "datasource": [
+        {
+          "uid": "gdev-testdata",
+          "type": "testdata"
+        },
+        {
+          "uid": "default.uid",
+          "type": "default.type"
+        }
+      ]
+    }
+  ],
+  "schemaVersion": 36,
+  "linkCount": 0,
+  "timeFrom": "now-6h",
+  "timeTo": "now",
+  "timezone": ""
+}

pkg/services/searchV2/extract/testdata/repeated-datasource-variables-with-default.json

@@ -0,0 +1,139 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 208,
+  "iteration": 1657048248365,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "testdata",
+        "uid": "${dsVariable}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto",
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 16,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "9.1.0-pre",
+      "repeat": "dsVariable",
+      "repeatDirection": "h",
+      "targets": [
+        {
+          "datasource": {
+            "type": "testdata",
+            "uid": "${dsVariable}"
+          },
+          "refId": "A"
+        }
+      ],
+      "title": "Panel Title",
+      "type": "table"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": [
+            "gdev-testdata",
+            "default"
+          ],
+          "value": [
+            "gdev-testdata",
+            "default"
+          ]
+        },
+        "hide": 0,
+        "includeAll": false,
+        "multi": true,
+        "name": "dsVariable",
+        "options": [],
+        "query": "testdata",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "new-dashboard-var-ds-test",
+  "uid": "2HLX_B3nk",
+  "version": 19,
+  "weekStart": ""
+}

pkg/services/searchV2/extract/testdata/repeated-datasource-variables.json

@@ -0,0 +1,143 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 209,
+  "iteration": 1656507534777,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "sqlite-datasource",
+        "uid": "${sqllite}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto",
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "9.1.0-pre",
+      "repeat": "sqllite",
+      "repeatDirection": "h",
+      "targets": [
+        {
+          "datasource": {
+            "type": "sqlite-datasource",
+            "uid": "${sqllite}"
+          },
+          "key": "Q-7ab4ab1a-3c70-41f4-9a45-19919e1c2193-0",
+          "queryText": "select * from user",
+          "queryType": "table",
+          "rawQueryText": "select * from user",
+          "refId": "A",
+          "timeColumns": [
+            "time",
+            "ts"
+          ]
+        }
+      ],
+      "title": "usersss!",
+      "type": "table"
+    }
+  ],
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": [
+            "SQLite Grafana",
+            "SQLite Grafana2"
+          ],
+          "value": [
+            "SQLite Grafana",
+            "SQLite Grafana2"
+          ]
+        },
+        "hide": 0,
+        "includeAll": false,
+        "multi": true,
+        "name": "sqllite",
+        "options": [],
+        "query": "sqlite-datasource",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "ds-variables",
+  "uid": "7BzWolqnk",
+  "version": 8,
+  "weekStart": ""
+}

pkg/services/searchV2/extract/testdata/string-datasource-variable-info.json

@@ -0,0 +1,33 @@
+{
+  "id": 209,
+  "title": "ds-variables",
+  "tags": null,
+  "templateVars": [
+    "sqllite"
+  ],
+  "datasource": [
+    {
+      "uid": "SQLite Grafana",
+      "type": "sqlite-datasource"
+    }
+  ],
+  "panels": [
+    {
+      "id": 1,
+      "title": "usersss!",
+      "type": "table",
+      "pluginVersion": "9.1.0-pre",
+      "datasource": [
+        {
+          "uid": "SQLite Grafana",
+          "type": "sqlite-datasource"
+        }
+      ]
+    }
+  ],
+  "schemaVersion": 36,
+  "linkCount": 0,
+  "timeFrom": "now-6h",
+  "timeTo": "now",
+  "timezone": ""
+}

pkg/services/searchV2/extract/testdata/string-datasource-variable.json

@@ -0,0 +1,109 @@
+{
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 209,
+  "iteration": 1656508852566,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": "${sqllite}",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto",
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "9.1.0-pre",
+      "targets": [
+        {
+          "datasource": {
+            "type": "sqlite-datasource",
+            "uid": "${sqllite}"
+          },
+          "queryText": "select * from user",
+          "queryType": "table",
+          "rawQueryText": "select * from user",
+          "refId": "A",
+          "timeColumns": [
+            "time",
+            "ts"
+          ]
+        }
+      ],
+      "title": "usersss!",
+      "type": "table"
+    }
+  ],
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "SQLite Grafana",
+          "value": "SQLite Grafana"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "name": "sqllite",
+        "options": [],
+        "query": "sqlite-datasource",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "ds-variables",
+  "uid": "7BzWolqnk",
+  "version": 13,
+  "weekStart": ""
+}

pkg/services/searchV2/extract/types.go

@@ -1,7 +1,10 @@
 package extract
 
-// empty everything will return the default
+type DatasourceLookup interface {
-type DatasourceLookup = func(ref *DataSourceRef) *DataSourceRef
+	// ByRef will return the default DS given empty reference (nil ref, or empty ref.uid and ref.type)
+	ByRef(ref *DataSourceRef) *DataSourceRef
+	ByType(dsType string) []DataSourceRef
+}
 
 type DataSourceRef struct {
 	UID  string `json:"uid,omitempty"`

pkg/services/searchV2/index.go

@@ -725,7 +725,7 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
 	}
 
 	// key will allow name or uid
-	lookup, err := loadDatasourceLookup(ctx, orgID, l.sql)
+	lookup, err := LoadDatasourceLookup(ctx, orgID, l.sql)
 	if err != nil {
 		return dashboards, err
 	}
@@ -821,13 +821,79 @@ type datasourceQueryResult struct {
 	IsDefault bool   `xorm:"is_default"`
 }
 
-func loadDatasourceLookup(ctx context.Context, orgID int64, sql *sqlstore.SQLStore) (extract.DatasourceLookup, error) {
+func createDatasourceLookup(rows []*datasourceQueryResult) extract.DatasourceLookup {
 	byUID := make(map[string]*extract.DataSourceRef, 50)
 	byName := make(map[string]*extract.DataSourceRef, 50)
+	byType := make(map[string][]extract.DataSourceRef, 50)
 	var defaultDS *extract.DataSourceRef
 
-	err := sql.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
+	for _, row := range rows {
-		rows := make([]*datasourceQueryResult, 0)
+		ref := &extract.DataSourceRef{
+			UID:  row.UID,
+			Type: row.Type,
+		}
+		byUID[row.UID] = ref
+		byName[row.Name] = ref
+		if row.IsDefault {
+			defaultDS = ref
+		}
+
+		if _, ok := byType[row.Type]; !ok {
+			byType[row.Type] = make([]extract.DataSourceRef, 5)
+		}
+		byType[row.Type] = append(byType[row.Type], *ref)
+	}
+
+	return &dsLookup{
+		byName:    byName,
+		byUID:     byUID,
+		byType:    byType,
+		defaultDS: defaultDS,
+	}
+}
+
+type dsLookup struct {
+	byName    map[string]*extract.DataSourceRef
+	byUID     map[string]*extract.DataSourceRef
+	byType    map[string][]extract.DataSourceRef
+	defaultDS *extract.DataSourceRef
+}
+
+func (d *dsLookup) ByRef(ref *extract.DataSourceRef) *extract.DataSourceRef {
+	if ref == nil {
+		return d.defaultDS
+	}
+	key := ""
+	if ref.UID != "" {
+		ds, ok := d.byUID[ref.UID]
+		if ok {
+			return ds
+		}
+		key = ref.UID
+	}
+	if key == "" {
+		return d.defaultDS
+	}
+	ds, ok := d.byUID[key]
+	if ok {
+		return ds
+	}
+	return d.byName[key]
+}
+
+func (d *dsLookup) ByType(dsType string) []extract.DataSourceRef {
+	ds, ok := d.byType[dsType]
+	if !ok {
+		return make([]extract.DataSourceRef, 0)
+	}
+
+	return ds
+}
+
+func LoadDatasourceLookup(ctx context.Context, orgID int64, sql *sqlstore.SQLStore) (extract.DatasourceLookup, error) {
+	rows := make([]*datasourceQueryResult, 0)
+
+	if err := sql.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
 		sess.Table("data_source").
 			Where("org_id = ?", orgID).
 			Cols("uid", "name", "type", "is_default")
@@ -837,44 +903,10 @@ func loadDatasourceLookup(ctx context.Context, orgID int64, sql *sqlstore.SQLSto
 			return err
 		}
 
-		for _, row := range rows {
-			ds := &extract.DataSourceRef{
-				UID:  row.UID,
-				Type: row.Type,
-			}
-			byUID[row.UID] = ds
-			byName[row.Name] = ds
-			if row.IsDefault {
-				defaultDS = ds
-			}
-		}
-
 		return nil
-	})
+	}); err != nil {
-	if err != nil {
 		return nil, err
 	}
 
-	// Lookup by UID or name
+	return createDatasourceLookup(rows), nil
-	return func(ref *extract.DataSourceRef) *extract.DataSourceRef {
-		if ref == nil {
-			return defaultDS
-		}
-		key := ""
-		if ref.UID != "" {
-			ds, ok := byUID[ref.UID]
-			if ok {
-				return ds
-			}
-			key = ref.UID
-		}
-		if key == "" {
-			return defaultDS
-		}
-		ds, ok := byUID[key]
-		if ok {
-			return ds
-		}
-		return byName[key]
-	}, err
 }