Search: support datasource template variables when parsing dashboard JSON models (#51587)

* init

* support template variables

* support variables without curly braces

* add todo for `__all` case

* fix `$__all` case for non-multivalue

* extract some functions

* fix flakinesss

* support `$__all` and `default` template variables

* add todo

* compilation fix

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
This commit is contained in:
Artur Wierzbicki 2022-07-08 01:59:24 +04:00 committed by GitHub
parent 2aff83d4e1
commit 63366615bb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 1948 additions and 112 deletions

View File

@ -11,11 +11,12 @@ import (
"github.com/google/uuid"
"github.com/grafana/grafana/pkg/infra/filestorage"
"github.com/grafana/grafana/pkg/services/searchV2"
"github.com/grafana/grafana/pkg/services/searchV2/extract"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
func exportDashboards(helper *commitHelper, job *gitExportJob, lookup dsLookup) error {
func exportDashboards(helper *commitHelper, job *gitExportJob) error {
alias := make(map[string]string, 100)
ids := make(map[int64]string, 100)
folders := make(map[int64]string, 100)
@ -25,13 +26,18 @@ func exportDashboards(helper *commitHelper, job *gitExportJob, lookup dsLookup)
folders[0] = job.cfg.GeneralFolderPath // "general"
}
lookup, err := searchV2.LoadDatasourceLookup(helper.ctx, helper.orgID, job.sql)
if err != nil {
return err
}
rootDir := path.Join(helper.orgDir, "root")
folderStructure := commitOptions{
when: time.Now(),
comment: "Exported folder structure",
}
err := job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
err = job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
type dashDataQueryResult struct {
Id int64
UID string `xorm:"uid"`

View File

@ -6,41 +6,26 @@ import (
"sort"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/searchV2/extract"
)
type dsLookup func(ref *extract.DataSourceRef) *extract.DataSourceRef
func exportDataSources(helper *commitHelper, job *gitExportJob, save bool) (dsLookup, error) {
func exportDataSources(helper *commitHelper, job *gitExportJob) error {
cmd := &datasources.GetDataSourcesQuery{
OrgId: job.orgID,
}
err := job.sql.GetDataSources(helper.ctx, cmd)
if err != nil {
return nil, err
return nil
}
sort.SliceStable(cmd.Result, func(i, j int) bool {
return cmd.Result[i].Created.After(cmd.Result[j].Created)
})
byUID := make(map[string]*extract.DataSourceRef, len(cmd.Result))
byName := make(map[string]*extract.DataSourceRef, len(cmd.Result))
for _, ds := range cmd.Result {
ref := &extract.DataSourceRef{
UID: ds.Uid,
Type: ds.Type,
}
byUID[ds.Uid] = ref
byName[ds.Name] = ref
if !save {
continue
}
ds.OrgId = 0
ds.Version = 0
ds.SecureJsonData = map[string][]byte{
"TODO": []byte("secret store lookup"),
"TODO": []byte("XXX"),
}
err := helper.add(commitOptions{
@ -54,26 +39,9 @@ func exportDataSources(helper *commitHelper, job *gitExportJob, save bool) (dsLo
comment: fmt.Sprintf("Add datasource: %s", ds.Name),
})
if err != nil {
return nil, err
return err
}
}
// Return the lookup function
return func(ref *extract.DataSourceRef) *extract.DataSourceRef {
if ref == nil || ref.UID == "" {
return &extract.DataSourceRef{
UID: "default.uid",
Type: "default.type",
}
}
v, ok := byUID[ref.UID]
if ok {
return v
}
v, ok = byName[ref.UID]
if ok {
return v
}
return nil
}, nil
return nil
}

View File

@ -172,19 +172,16 @@ func (e *gitExportJob) doExportWithHistory() error {
func (e *gitExportJob) doOrgExportWithHistory(helper *commitHelper) error {
include := e.cfg.Include
lookup, err := exportDataSources(helper, e, include.DS)
if err != nil {
return err
}
if include.Dash {
err = exportDashboards(helper, e, lookup)
if err != nil {
return err
}
}
exporters := []simpleExporter{}
if include.Dash {
exporters = append(exporters, exportDashboards)
}
if include.DS {
exporters = append(exporters, exportDataSources)
}
if include.Auth {
exporters = append(exporters, dumpAuthTables)
}
@ -207,12 +204,12 @@ func (e *gitExportJob) doOrgExportWithHistory(helper *commitHelper) error {
}
for _, fn := range exporters {
err = fn(helper, e)
err := fn(helper, e)
if err != nil {
return err
}
}
return err
return nil
}
/**

View File

@ -3,6 +3,7 @@ package extract
import (
"io"
"strconv"
"strings"
jsoniter "github.com/json-iterator/go"
)
@ -11,6 +12,99 @@ func logf(format string, a ...interface{}) {
//fmt.Printf(format, a...)
}
type templateVariable struct {
current struct {
value interface{}
}
name string
query interface{}
variableType string
}
type datasourceVariableLookup struct {
variableNameToRefs map[string][]DataSourceRef
dsLookup DatasourceLookup
}
func (d *datasourceVariableLookup) getDsRefsByTemplateVariableValue(value string, datasourceType string) []DataSourceRef {
switch value {
case "default":
// can be the default DS, or a DS with UID="default"
candidateDs := d.dsLookup.ByRef(&DataSourceRef{UID: value})
if candidateDs == nil {
// get the actual default DS
candidateDs = d.dsLookup.ByRef(nil)
}
return []DataSourceRef{*candidateDs}
case "$__all":
// TODO: filter datasources by template variable's regex
return d.dsLookup.ByType(datasourceType)
case "":
return []DataSourceRef{}
case "No data sources found":
return []DataSourceRef{}
default:
return []DataSourceRef{
{
UID: value,
Type: datasourceType,
},
}
}
}
func (d *datasourceVariableLookup) add(templateVariable templateVariable) {
var refs []DataSourceRef
datasourceType, isDataSourceTypeValid := templateVariable.query.(string)
if !isDataSourceTypeValid {
d.variableNameToRefs[templateVariable.name] = refs
return
}
if values, multiValueVariable := templateVariable.current.value.([]interface{}); multiValueVariable {
for _, value := range values {
if valueAsString, ok := value.(string); ok {
refs = append(refs, d.getDsRefsByTemplateVariableValue(valueAsString, datasourceType)...)
}
}
}
if value, stringValue := templateVariable.current.value.(string); stringValue {
refs = append(refs, d.getDsRefsByTemplateVariableValue(value, datasourceType)...)
}
d.variableNameToRefs[templateVariable.name] = unique(refs)
}
func unique(refs []DataSourceRef) []DataSourceRef {
var uniqueRefs []DataSourceRef
uidPresence := make(map[string]bool)
for _, ref := range refs {
if !uidPresence[ref.UID] {
uidPresence[ref.UID] = true
uniqueRefs = append(uniqueRefs, ref)
}
}
return uniqueRefs
}
func (d *datasourceVariableLookup) getDatasourceRefs(name string) []DataSourceRef {
refs, ok := d.variableNameToRefs[name]
if ok {
return refs
}
return []DataSourceRef{}
}
func newDatasourceVariableLookup(dsLookup DatasourceLookup) *datasourceVariableLookup {
return &datasourceVariableLookup{
variableNameToRefs: make(map[string][]DataSourceRef),
dsLookup: dsLookup,
}
}
// nolint:gocyclo
// ReadDashboard will take a byte stream and return dashboard info
func ReadDashboard(stream io.Reader, lookup DatasourceLookup) (*DashboardInfo, error) {
@ -18,6 +112,8 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) (*DashboardInfo, e
iter := jsoniter.Parse(jsoniter.ConfigDefault, stream, 1024)
datasourceVariablesLookup := newDatasourceVariableLookup(lookup)
for l1Field := iter.ReadObject(); l1Field != ""; l1Field = iter.ReadObject() {
// Skip null values so we don't need special int handling
if iter.WhatIsNext() == jsoniter.NilValue {
@ -112,13 +208,34 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) (*DashboardInfo, e
for sub := iter.ReadObject(); sub != ""; sub = iter.ReadObject() {
if sub == "list" {
for iter.ReadArray() {
templateVariable := templateVariable{}
for k := iter.ReadObject(); k != ""; k = iter.ReadObject() {
if k == "name" {
dash.TemplateVars = append(dash.TemplateVars, iter.ReadString())
} else {
switch k {
case "name":
name := iter.ReadString()
dash.TemplateVars = append(dash.TemplateVars, name)
templateVariable.name = name
case "type":
templateVariable.variableType = iter.ReadString()
case "query":
templateVariable.query = iter.Read()
case "current":
for c := iter.ReadObject(); c != ""; c = iter.ReadObject() {
if c == "value" {
templateVariable.current.value = iter.Read()
} else {
iter.Skip()
}
}
default:
iter.Skip()
}
}
if templateVariable.variableType == "datasource" {
datasourceVariablesLookup.add(templateVariable)
}
}
} else {
iter.Skip()
@ -139,6 +256,8 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) (*DashboardInfo, e
}
}
replaceDatasourceVariables(dash, datasourceVariablesLookup)
targets := newTargetInfo(lookup)
for _, panel := range dash.Panels {
targets.addPanel(panel)
@ -148,6 +267,43 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) (*DashboardInfo, e
return dash, iter.Error
}
func replaceDatasourceVariables(dash *DashboardInfo, datasourceVariablesLookup *datasourceVariableLookup) {
for i, panel := range dash.Panels {
var dsVariableRefs []DataSourceRef
var dsRefs []DataSourceRef
// partition into actual datasource references and variables
for i := range panel.Datasource {
isVariableRef := strings.HasPrefix(panel.Datasource[i].UID, "$")
if isVariableRef {
dsVariableRefs = append(dsVariableRefs, panel.Datasource[i])
} else {
dsRefs = append(dsRefs, panel.Datasource[i])
}
}
dash.Panels[i].Datasource = append(dsRefs, findDatasourceRefsForVariables(dsVariableRefs, datasourceVariablesLookup)...)
}
}
func getDataSourceVariableName(dsVariableRef DataSourceRef) string {
if strings.HasPrefix(dsVariableRef.UID, "${") {
return strings.TrimPrefix(strings.TrimSuffix(dsVariableRef.UID, "}"), "${")
}
return strings.TrimPrefix(dsVariableRef.UID, "$")
}
func findDatasourceRefsForVariables(dsVariableRefs []DataSourceRef, datasourceVariablesLookup *datasourceVariableLookup) []DataSourceRef {
var referencedDs []DataSourceRef
for _, dsVariableRef := range dsVariableRefs {
variableName := getDataSourceVariableName(dsVariableRef)
refs := datasourceVariablesLookup.getDatasourceRefs(variableName)
referencedDs = append(referencedDs, refs...)
}
return referencedDs
}
// will always return strings for now
func readPanelInfo(iter *jsoniter.Iterator, lookup DatasourceLookup) PanelInfo {
panel := PanelInfo{}

View File

@ -4,28 +4,61 @@ import (
"encoding/json"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type dsLookup struct {
}
func (d *dsLookup) ByRef(ref *DataSourceRef) *DataSourceRef {
if ref == nil || ref.UID == "" {
return &DataSourceRef{
UID: "default.uid",
Type: "default.type",
}
}
if ref.UID == "default" {
return nil
}
return ref
}
func (d *dsLookup) ByType(dsType string) []DataSourceRef {
if dsType == "sqlite-datasource" {
return []DataSourceRef{
{
UID: "sqlite-1",
Type: "sqlite-datasource",
},
{
UID: "sqlite-2",
Type: "sqlite-datasource",
},
}
}
return make([]DataSourceRef, 0)
}
func TestReadDashboard(t *testing.T) {
inputs := []string{
"check-string-datasource-id",
"all-panels",
"panel-graph/graph-shared-tooltips",
}
// key will allow name or uid
ds := func(ref *DataSourceRef) *DataSourceRef {
if ref == nil || ref.UID == "" {
return &DataSourceRef{
UID: "default.uid",
Type: "default.type",
}
}
return ref
"datasource-variable",
"default-datasource-variable",
"empty-datasource-variable",
"repeated-datasource-variables",
"string-datasource-variable",
"datasource-variable-no-curly-braces",
"all-selected-multi-datasource-variable",
"all-selected-single-datasource-variable",
"repeated-datasource-variables-with-default",
}
devdash := "../../../../devenv/dev-dashboards/"
@ -44,7 +77,9 @@ func TestReadDashboard(t *testing.T) {
}
require.NoError(t, err)
dash, err := ReadDashboard(f, ds)
dash, err := ReadDashboard(f, &dsLookup{})
sortDatasources(dash)
require.NoError(t, err)
out, err := json.MarshalIndent(dash, "", " ")
require.NoError(t, err)
@ -64,3 +99,16 @@ func TestReadDashboard(t *testing.T) {
}
}
}
// assure consistent ordering of datasources to prevent random failures of `assert.JSONEq`
func sortDatasources(dash *DashboardInfo) {
sort.Slice(dash.Datasource, func(i, j int) bool {
return strings.Compare(dash.Datasource[i].UID, dash.Datasource[j].UID) > 0
})
for panelId := range dash.Panels {
sort.Slice(dash.Panels[panelId].Datasource, func(i, j int) bool {
return strings.Compare(dash.Panels[panelId].Datasource[i].UID, dash.Panels[panelId].Datasource[j].UID) > 0
})
}
}

View File

@ -31,17 +31,17 @@ func (s *targetInfo) addDatasource(iter *jsoniter.Iterator) {
switch iter.WhatIsNext() {
case jsoniter.StringValue:
key := iter.ReadString()
ds := s.lookup(&DataSourceRef{UID: key})
ds := s.lookup.ByRef(&DataSourceRef{UID: key})
s.addRef(ds)
case jsoniter.NilValue:
s.addRef(s.lookup(nil))
s.addRef(s.lookup.ByRef(nil))
iter.Skip()
case jsoniter.ObjectValue:
ref := &DataSourceRef{}
iter.ReadVal(ref)
ds := s.lookup(ref)
ds := s.lookup.ByRef(ref)
s.addRef(ds)
default:

View File

@ -0,0 +1,61 @@
{
"id": 209,
"title": "ds-variables",
"tags": null,
"templateVars": [
"sqllite"
],
"datasource": [
{
"uid": "sqlite-2",
"type": "sqlite-datasource"
},
{
"uid": "sqlite-1",
"type": "sqlite-datasource"
}
],
"panels": [
{
"id": 7,
"title": "Panel Title",
"type": "timeseries",
"datasource": [
{
"uid": "sqlite-2",
"type": "sqlite-datasource"
},
{
"uid": "sqlite-1",
"type": "sqlite-datasource"
}
]
},
{
"id": 3,
"title": "Row title",
"type": "row"
},
{
"id": 1,
"title": "usersss!",
"type": "table",
"pluginVersion": "9.1.0-pre",
"datasource": [
{
"uid": "sqlite-2",
"type": "sqlite-datasource"
},
{
"uid": "sqlite-1",
"type": "sqlite-datasource"
}
]
}
],
"schemaVersion": 36,
"linkCount": 0,
"timeFrom": "now-6h",
"timeTo": "now",
"timezone": ""
}

View File

@ -0,0 +1,230 @@
{
"annotations": {
"list": [
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 209,
"iteration": 1656533909544,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "sqlite-datasource",
"uid": "${sqllite}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 7,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"repeat": "sqllite",
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "sqlite-datasource",
"uid": "${sqllite}"
},
"queryText": "\n SELECT CAST(strftime('%s', 'now', '-1 minute') as INTEGER) as time, 4 as value\n WHERE time >= 1234 and time < 134567\n ",
"queryType": "table",
"rawQueryText": "SELECT CAST(strftime('%s', 'now', '-1 minute') as INTEGER) as time, 4 as value \nWHERE time >= $__from / 1000 and time < $__to / 1000",
"refId": "A",
"timeColumns": [
"time",
"ts"
]
}
],
"title": "Panel Title",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 8
},
"id": 3,
"panels": [],
"repeat": "sqllite",
"repeatDirection": "h",
"title": "Row title",
"type": "row"
},
{
"datasource": {
"type": "sqlite-datasource",
"uid": "${sqllite}"
},
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 8,
"y": 9
},
"id": 1,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.1.0-pre",
"targets": [
{
"datasource": {
"type": "sqlite-datasource",
"uid": "${sqllite}"
},
"queryText": "select * from user",
"queryType": "table",
"rawQueryText": "select * from user",
"refId": "A",
"timeColumns": [
"time",
"ts"
]
}
],
"title": "usersss!",
"type": "table"
}
],
"schemaVersion": 36,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"allValue": "",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"hide": 0,
"includeAll": true,
"multi": true,
"name": "sqllite",
"options": [],
"query": "sqlite-datasource",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "ds-variables",
"uid": "7BzWolqnk",
"version": 36,
"weekStart": ""
}

View File

@ -0,0 +1,41 @@
{
"id": 208,
"title": "new-dashboard-var-ds-test",
"tags": null,
"templateVars": [
"dsVariable"
],
"datasource": [
{
"uid": "sqlite-2",
"type": "sqlite-datasource"
},
{
"uid": "sqlite-1",
"type": "sqlite-datasource"
}
],
"panels": [
{
"id": 2,
"title": "Panel Title",
"type": "table",
"pluginVersion": "9.1.0-pre",
"datasource": [
{
"uid": "sqlite-2",
"type": "sqlite-datasource"
},
{
"uid": "sqlite-1",
"type": "sqlite-datasource"
}
]
}
],
"schemaVersion": 36,
"linkCount": 0,
"timeFrom": "now-6h",
"timeTo": "now",
"timezone": ""
}

View File

@ -0,0 +1,133 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 208,
"iteration": 1657048248371,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "sqlite-datasource",
"uid": "${dsVariable}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 16,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.1.0-pre",
"repeat": "dsVariable",
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "sqlite-datasource",
"uid": "${dsVariable}"
},
"refId": "A"
}
],
"title": "Panel Title",
"type": "table"
}
],
"refresh": "",
"schemaVersion": 36,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"hide": 0,
"includeAll": true,
"multi": false,
"name": "dsVariable",
"options": [],
"query": "sqlite-datasource",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "new-dashboard-var-ds-test",
"uid": "2HLX_B3nk",
"version": 21,
"weekStart": ""
}

View File

@ -0,0 +1,33 @@
{
"id": 209,
"title": "ds-variables",
"tags": null,
"templateVars": [
"sqllite"
],
"datasource": [
{
"uid": "SQLite Grafana",
"type": "sqlite-datasource"
}
],
"panels": [
{
"id": 1,
"title": "usersss!",
"type": "table",
"pluginVersion": "9.1.0-pre",
"datasource": [
{
"uid": "SQLite Grafana",
"type": "sqlite-datasource"
}
]
}
],
"schemaVersion": 36,
"linkCount": 0,
"timeFrom": "now-6h",
"timeTo": "now",
"timezone": ""
}

View File

@ -0,0 +1,33 @@
{
"id": 209,
"title": "ds-variables",
"tags": null,
"templateVars": [
"sqllite"
],
"datasource": [
{
"uid": "SQLite Grafana",
"type": "sqlite-datasource"
}
],
"panels": [
{
"id": 1,
"title": "usersss!",
"type": "table",
"pluginVersion": "9.1.0-pre",
"datasource": [
{
"uid": "SQLite Grafana",
"type": "sqlite-datasource"
}
]
}
],
"schemaVersion": 36,
"linkCount": 0,
"timeFrom": "now-6h",
"timeTo": "now",
"timezone": ""
}

View File

@ -0,0 +1,134 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 209,
"iteration": 1656508852566,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "sqlite-datasource",
"uid": "$sqllite"
},
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.1.0-pre",
"targets": [
{
"datasource": {
"type": "sqlite-datasource",
"uid": "$sqllite"
},
"queryText": "select * from user",
"queryType": "table",
"rawQueryText": "select * from user",
"refId": "A",
"timeColumns": [
"time",
"ts"
]
}
],
"title": "usersss!",
"type": "table"
}
],
"schemaVersion": 36,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "SQLite Grafana",
"value": "SQLite Grafana"
},
"hide": 0,
"includeAll": false,
"multi": false,
"name": "sqllite",
"options": [],
"query": "sqlite-datasource",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "ds-variables",
"uid": "7BzWolqnk",
"version": 13,
"weekStart": ""
}

View File

@ -0,0 +1,134 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 209,
"iteration": 1656508852566,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "sqlite-datasource",
"uid": "${sqllite}"
},
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.1.0-pre",
"targets": [
{
"datasource": {
"type": "sqlite-datasource",
"uid": "${sqllite}"
},
"queryText": "select * from user",
"queryType": "table",
"rawQueryText": "select * from user",
"refId": "A",
"timeColumns": [
"time",
"ts"
]
}
],
"title": "usersss!",
"type": "table"
}
],
"schemaVersion": 36,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "SQLite Grafana",
"value": "SQLite Grafana"
},
"hide": 0,
"includeAll": false,
"multi": false,
"name": "sqllite",
"options": [],
"query": "sqlite-datasource",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "ds-variables",
"uid": "7BzWolqnk",
"version": 13,
"weekStart": ""
}

View File

@ -0,0 +1,33 @@
{
"id": 208,
"title": "new-dashboard-var-ds-test",
"tags": null,
"templateVars": [
"dsVariable"
],
"datasource": [
{
"uid": "default.uid",
"type": "default.type"
}
],
"panels": [
{
"id": 2,
"title": "Panel Title",
"type": "table",
"pluginVersion": "9.1.0-pre",
"datasource": [
{
"uid": "default.uid",
"type": "default.type"
}
]
}
],
"schemaVersion": 36,
"linkCount": 0,
"timeFrom": "now-6h",
"timeTo": "now",
"timezone": ""
}

View File

@ -0,0 +1,133 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 208,
"iteration": 1657048248373,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "testdata",
"uid": "${dsVariable}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 16,
"w": 24,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.1.0-pre",
"repeat": "dsVariable",
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "testdata",
"uid": "${dsVariable}"
},
"refId": "A"
}
],
"title": "Panel Title",
"type": "table"
}
],
"refresh": "",
"schemaVersion": 36,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": true,
"text": "default",
"value": "default"
},
"hide": 0,
"includeAll": false,
"multi": false,
"name": "dsVariable",
"options": [],
"query": "testdata",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "new-dashboard-var-ds-test",
"uid": "2HLX_B3nk",
"version": 22,
"weekStart": ""
}

View File

@ -0,0 +1,21 @@
{
"id": 209,
"title": "ds-variables",
"tags": null,
"templateVars": [
"sqllite"
],
"panels": [
{
"id": 1,
"title": "usersss!",
"type": "table",
"pluginVersion": "9.1.0-pre"
}
],
"schemaVersion": 36,
"linkCount": 0,
"timeFrom": "now-6h",
"timeTo": "now",
"timezone": ""
}

View File

@ -0,0 +1,134 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 209,
"iteration": 1656513278471,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "sqlite-datasource",
"uid": "${sqllite}"
},
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.1.0-pre",
"targets": [
{
"datasource": {
"type": "sqlite-datasource",
"uid": "${sqllite}"
},
"queryText": "select * from user",
"queryType": "table",
"rawQueryText": "select * from user",
"refId": "A",
"timeColumns": [
"time",
"ts"
]
}
],
"title": "usersss!",
"type": "table"
}
],
"schemaVersion": 36,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "No data sources found",
"value": ""
},
"hide": 0,
"includeAll": false,
"multi": false,
"name": "sqllite",
"options": [],
"query": "sqlite-datasource",
"queryValue": "",
"refresh": 1,
"regex": "asdgasd",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "ds-variables",
"uid": "7BzWolqnk",
"version": 14,
"weekStart": ""
}

View File

@ -0,0 +1,41 @@
{
"id": 209,
"title": "ds-variables",
"tags": null,
"templateVars": [
"sqllite"
],
"datasource": [
{
"uid": "SQLite Grafana2",
"type": "sqlite-datasource"
},
{
"uid": "SQLite Grafana",
"type": "sqlite-datasource"
}
],
"panels": [
{
"id": 1,
"title": "usersss!",
"type": "table",
"pluginVersion": "9.1.0-pre",
"datasource": [
{
"uid": "SQLite Grafana2",
"type": "sqlite-datasource"
},
{
"uid": "SQLite Grafana",
"type": "sqlite-datasource"
}
]
}
],
"schemaVersion": 36,
"linkCount": 0,
"timeFrom": "now-6h",
"timeTo": "now",
"timezone": ""
}

View File

@ -0,0 +1,41 @@
{
"id": 208,
"title": "new-dashboard-var-ds-test",
"tags": null,
"templateVars": [
"dsVariable"
],
"datasource": [
{
"uid": "gdev-testdata",
"type": "testdata"
},
{
"uid": "default.uid",
"type": "default.type"
}
],
"panels": [
{
"id": 2,
"title": "Panel Title",
"type": "table",
"pluginVersion": "9.1.0-pre",
"datasource": [
{
"uid": "gdev-testdata",
"type": "testdata"
},
{
"uid": "default.uid",
"type": "default.type"
}
]
}
],
"schemaVersion": 36,
"linkCount": 0,
"timeFrom": "now-6h",
"timeTo": "now",
"timezone": ""
}

View File

@ -0,0 +1,139 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 208,
"iteration": 1657048248365,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "testdata",
"uid": "${dsVariable}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 16,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.1.0-pre",
"repeat": "dsVariable",
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "testdata",
"uid": "${dsVariable}"
},
"refId": "A"
}
],
"title": "Panel Title",
"type": "table"
}
],
"refresh": "",
"schemaVersion": 36,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": true,
"text": [
"gdev-testdata",
"default"
],
"value": [
"gdev-testdata",
"default"
]
},
"hide": 0,
"includeAll": false,
"multi": true,
"name": "dsVariable",
"options": [],
"query": "testdata",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "new-dashboard-var-ds-test",
"uid": "2HLX_B3nk",
"version": 19,
"weekStart": ""
}

View File

@ -0,0 +1,143 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 209,
"iteration": 1656507534777,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "sqlite-datasource",
"uid": "${sqllite}"
},
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.1.0-pre",
"repeat": "sqllite",
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "sqlite-datasource",
"uid": "${sqllite}"
},
"key": "Q-7ab4ab1a-3c70-41f4-9a45-19919e1c2193-0",
"queryText": "select * from user",
"queryType": "table",
"rawQueryText": "select * from user",
"refId": "A",
"timeColumns": [
"time",
"ts"
]
}
],
"title": "usersss!",
"type": "table"
}
],
"schemaVersion": 36,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": [
"SQLite Grafana",
"SQLite Grafana2"
],
"value": [
"SQLite Grafana",
"SQLite Grafana2"
]
},
"hide": 0,
"includeAll": false,
"multi": true,
"name": "sqllite",
"options": [],
"query": "sqlite-datasource",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "ds-variables",
"uid": "7BzWolqnk",
"version": 8,
"weekStart": ""
}

View File

@ -0,0 +1,33 @@
{
"id": 209,
"title": "ds-variables",
"tags": null,
"templateVars": [
"sqllite"
],
"datasource": [
{
"uid": "SQLite Grafana",
"type": "sqlite-datasource"
}
],
"panels": [
{
"id": 1,
"title": "usersss!",
"type": "table",
"pluginVersion": "9.1.0-pre",
"datasource": [
{
"uid": "SQLite Grafana",
"type": "sqlite-datasource"
}
]
}
],
"schemaVersion": 36,
"linkCount": 0,
"timeFrom": "now-6h",
"timeTo": "now",
"timezone": ""
}

View File

@ -0,0 +1,109 @@
{
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 209,
"iteration": 1656508852566,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": "${sqllite}",
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.1.0-pre",
"targets": [
{
"datasource": {
"type": "sqlite-datasource",
"uid": "${sqllite}"
},
"queryText": "select * from user",
"queryType": "table",
"rawQueryText": "select * from user",
"refId": "A",
"timeColumns": [
"time",
"ts"
]
}
],
"title": "usersss!",
"type": "table"
}
],
"schemaVersion": 36,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "SQLite Grafana",
"value": "SQLite Grafana"
},
"hide": 0,
"includeAll": false,
"multi": false,
"name": "sqllite",
"options": [],
"query": "sqlite-datasource",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "ds-variables",
"uid": "7BzWolqnk",
"version": 13,
"weekStart": ""
}

View File

@ -1,7 +1,10 @@
package extract
// empty everything will return the default
type DatasourceLookup = func(ref *DataSourceRef) *DataSourceRef
type DatasourceLookup interface {
// ByRef will return the default DS given empty reference (nil ref, or empty ref.uid and ref.type)
ByRef(ref *DataSourceRef) *DataSourceRef
ByType(dsType string) []DataSourceRef
}
type DataSourceRef struct {
UID string `json:"uid,omitempty"`

View File

@ -725,7 +725,7 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
}
// key will allow name or uid
lookup, err := loadDatasourceLookup(ctx, orgID, l.sql)
lookup, err := LoadDatasourceLookup(ctx, orgID, l.sql)
if err != nil {
return dashboards, err
}
@ -821,13 +821,79 @@ type datasourceQueryResult struct {
IsDefault bool `xorm:"is_default"`
}
func loadDatasourceLookup(ctx context.Context, orgID int64, sql *sqlstore.SQLStore) (extract.DatasourceLookup, error) {
func createDatasourceLookup(rows []*datasourceQueryResult) extract.DatasourceLookup {
byUID := make(map[string]*extract.DataSourceRef, 50)
byName := make(map[string]*extract.DataSourceRef, 50)
byType := make(map[string][]extract.DataSourceRef, 50)
var defaultDS *extract.DataSourceRef
err := sql.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
rows := make([]*datasourceQueryResult, 0)
for _, row := range rows {
ref := &extract.DataSourceRef{
UID: row.UID,
Type: row.Type,
}
byUID[row.UID] = ref
byName[row.Name] = ref
if row.IsDefault {
defaultDS = ref
}
if _, ok := byType[row.Type]; !ok {
byType[row.Type] = make([]extract.DataSourceRef, 5)
}
byType[row.Type] = append(byType[row.Type], *ref)
}
return &dsLookup{
byName: byName,
byUID: byUID,
byType: byType,
defaultDS: defaultDS,
}
}
type dsLookup struct {
byName map[string]*extract.DataSourceRef
byUID map[string]*extract.DataSourceRef
byType map[string][]extract.DataSourceRef
defaultDS *extract.DataSourceRef
}
func (d *dsLookup) ByRef(ref *extract.DataSourceRef) *extract.DataSourceRef {
if ref == nil {
return d.defaultDS
}
key := ""
if ref.UID != "" {
ds, ok := d.byUID[ref.UID]
if ok {
return ds
}
key = ref.UID
}
if key == "" {
return d.defaultDS
}
ds, ok := d.byUID[key]
if ok {
return ds
}
return d.byName[key]
}
func (d *dsLookup) ByType(dsType string) []extract.DataSourceRef {
ds, ok := d.byType[dsType]
if !ok {
return make([]extract.DataSourceRef, 0)
}
return ds
}
func LoadDatasourceLookup(ctx context.Context, orgID int64, sql *sqlstore.SQLStore) (extract.DatasourceLookup, error) {
rows := make([]*datasourceQueryResult, 0)
if err := sql.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
sess.Table("data_source").
Where("org_id = ?", orgID).
Cols("uid", "name", "type", "is_default")
@ -837,44 +903,10 @@ func loadDatasourceLookup(ctx context.Context, orgID int64, sql *sqlstore.SQLSto
return err
}
for _, row := range rows {
ds := &extract.DataSourceRef{
UID: row.UID,
Type: row.Type,
}
byUID[row.UID] = ds
byName[row.Name] = ds
if row.IsDefault {
defaultDS = ds
}
}
return nil
})
if err != nil {
}); err != nil {
return nil, err
}
// Lookup by UID or name
return func(ref *extract.DataSourceRef) *extract.DataSourceRef {
if ref == nil {
return defaultDS
}
key := ""
if ref.UID != "" {
ds, ok := byUID[ref.UID]
if ok {
return ds
}
key = ref.UID
}
if key == "" {
return defaultDS
}
ds, ok := byUID[key]
if ok {
return ds
}
return byName[key]
}, err
return createDatasourceLookup(rows), nil
}