Provisioning: Support FolderUid in Dashboard Provisioning Config (#16559)

* add folderUid to DashbaordsAsConfig structs and DashbardProviderConfigs struct, set these values in mapping func
look for new folderUid values in config_reader tests
set dashboard folder Uid explicitly in file_reader, which has no affect when not given

* formatting and docstrings

* add folderUid to DashbaordsAsConfig structs and DashbardProviderConfigs struct, set these values in mapping func
look for new folderUid values in config_reader tests
set dashboard folder Uid explicitly in file_reader, which has no affect when not given

* formatting and docstrings

* add folderUid option, as well as documentation for the rest of the fields

* add blank folderUid in devenv example.

* add folderUid to provisioning sample yaml

* instead of just warning, return error if unmarshalling dashboard provisioning file fails

* Removing the error handling and adding comment

* Add duplicity check for folder Uids


Co-authored-by: swtch1 <joshua.thornton@protonmail.com>

conf/provisioning/dashboards/sample.yaml

@@ -5,6 +5,7 @@ apiVersion: 1
 # - name: 'default'
 #   orgId: 1
 #   folder: ''
+#   folderUid: ''
 #   type: file
 #   options:
 #     path: /var/lib/grafana/dashboards

devenv/dashboards.yaml

@@ -3,6 +3,7 @@ apiVersion: 1
 providers:
  - name: 'gdev dashboards'
    folder: 'gdev dashboards'
+   folderUid: ''
    type: file
    updateIntervalSeconds: 60
    options:

docs/sources/administration/provisioning.md

@@ -203,13 +203,24 @@ The dashboard provider config file looks somewhat like this:
 apiVersion: 1
 
 providers:
+  # <string> provider name
 - name: 'default'
+  # <int> org id. will default to orgId 1 if not specified
   orgId: 1
+  # <string, required> name of the dashboard folder. Required
   folder: ''
+  # <string> folder UID. will be automatically generated if not specified
+  folderUid: ''
+  # <string, required> provider type. Required
   type: file
+  # <bool> disable dashboard deletion
   disableDeletion: false
-  updateIntervalSeconds: 10 #how often Grafana will scan for changed dashboards
+  # <bool> enable dashboard editing
+  editable: true
+  # <int> how often Grafana will scan for changed dashboards
+  updateIntervalSeconds: 10  
   options:
+    # <string, required> path to dashboard files on disk. Required
     path: /var/lib/grafana/dashboards
 ```
 

pkg/services/provisioning/dashboards/config_reader.go

@@ -24,10 +24,15 @@ func (cr *configReader) parseConfigs(file os.FileInfo) ([]*DashboardsAsConfig, e
 	}
 
 	apiVersion := &ConfigVersion{ApiVersion: 0}
-	yaml.Unmarshal(yamlFile, &apiVersion)
+
+	// We ignore the error here because it errors out for version 0 which does not have apiVersion
+	// specified (so 0 is default). This can also error in case the apiVersion is not an integer but at the moment
+	// this does not handle that case and would still go on as if version = 0.
+	// TODO: return appropriate error in case the apiVersion is specified but isn't integer (or even if it is
+	//  integer > max version?).
+	_ = yaml.Unmarshal(yamlFile, &apiVersion)
 
 	if apiVersion.ApiVersion > 0 {
-
 		v1 := &DashboardAsConfigV1{}
 		err := yaml.Unmarshal(yamlFile, &v1)
 		if err != nil {
@@ -37,7 +42,6 @@ func (cr *configReader) parseConfigs(file os.FileInfo) ([]*DashboardsAsConfig, e
 		if v1 != nil {
 			return v1.mapToDashboardAsConfig(), nil
 		}
-
 	} else {
 		var v0 []*DashboardsAsConfigV0
 		err := yaml.Unmarshal(yamlFile, &v0)
@@ -78,13 +82,23 @@ func (cr *configReader) readConfig() ([]*DashboardsAsConfig, error) {
 		}
 	}
 
-	for i := range dashboards {
+	uidUsage := map[string]uint8{}
-		if dashboards[i].OrgId == 0 {
+	for _, dashboard := range dashboards {
-			dashboards[i].OrgId = 1
+		if dashboard.OrgId == 0 {
+			dashboard.OrgId = 1
 		}
 
-		if dashboards[i].UpdateIntervalSeconds == 0 {
+		if dashboard.UpdateIntervalSeconds == 0 {
-			dashboards[i].UpdateIntervalSeconds = 10
+			dashboard.UpdateIntervalSeconds = 10
+		}
+		if len(dashboard.FolderUid) > 0 {
+			uidUsage[dashboard.FolderUid] += 1
+		}
+	}
+
+	for uid, times := range uidUsage {
+		if times > 1 {
+			cr.log.Error("the same 'folderUid' is used more than once", "folderUid", uid)
 		}
 	}
 

pkg/services/provisioning/dashboards/config_reader_test.go

@@ -66,6 +66,7 @@ func validateDashboardAsConfig(t *testing.T, cfg []*DashboardsAsConfig) {
 	So(ds.Type, ShouldEqual, "file")
 	So(ds.OrgId, ShouldEqual, 2)
 	So(ds.Folder, ShouldEqual, "developers")
+	So(ds.FolderUid, ShouldEqual, "xyz")
 	So(ds.Editable, ShouldBeTrue)
 	So(len(ds.Options), ShouldEqual, 1)
 	So(ds.Options["path"], ShouldEqual, "/var/lib/grafana/dashboards")
@@ -77,6 +78,7 @@ func validateDashboardAsConfig(t *testing.T, cfg []*DashboardsAsConfig) {
 	So(ds2.Type, ShouldEqual, "file")
 	So(ds2.OrgId, ShouldEqual, 1)
 	So(ds2.Folder, ShouldEqual, "")
+	So(ds2.FolderUid, ShouldEqual, "")
 	So(ds2.Editable, ShouldBeFalse)
 	So(len(ds2.Options), ShouldEqual, 1)
 	So(ds2.Options["path"], ShouldEqual, "/var/lib/grafana/dashboards")

pkg/services/provisioning/dashboards/file_reader.go

@@ -78,6 +78,7 @@ func (fr *fileReader) ReadAndListen(ctx context.Context) error {
 	}
 }
 
+// startWalkingDisk finds and saves dashboards on disk.
 func (fr *fileReader) startWalkingDisk() error {
 	resolvedPath := fr.resolvePath(fr.Path)
 	if _, err := os.Stat(resolvedPath); err != nil {
@@ -119,6 +120,7 @@ func (fr *fileReader) startWalkingDisk() error {
 	return nil
 }
 
+// handleMissingDashboardFiles will unprovision or delete dashboards which are missing on disk.
 func (fr *fileReader) handleMissingDashboardFiles(provisionedDashboardRefs map[string]*models.DashboardProvisioning, filesFoundOnDisk map[string]os.FileInfo) {
 	// find dashboards to delete since json file is missing
 	var dashboardToDelete []int64
@@ -151,6 +153,7 @@ func (fr *fileReader) handleMissingDashboardFiles(provisionedDashboardRefs map[s
 	}
 }
 
+// saveDashboard saves or updates the dashboard provisioning file at path.
 func (fr *fileReader) saveDashboard(path string, folderId int64, fileInfo os.FileInfo, provisionedDashboardRefs map[string]*models.DashboardProvisioning) (provisioningMetadata, error) {
 	provisioningMetadata := provisioningMetadata{}
 	resolvedFileInfo, err := resolveSymlink(fileInfo, path)
@@ -189,7 +192,7 @@ func (fr *fileReader) saveDashboard(path string, folderId int64, fileInfo os.Fil
 		dash.Dashboard.SetId(provisionedData.DashboardId)
 	}
 
-	fr.log.Debug("saving new dashboard", "provisoner", fr.Cfg.Name, "file", path, "folderId", dash.Dashboard.FolderId)
+	fr.log.Debug("saving new dashboard", "provisioner", fr.Cfg.Name, "file", path, "folderId", dash.Dashboard.FolderId)
 	dp := &models.DashboardProvisioning{
 		ExternalId: path,
 		Name:       fr.Cfg.Name,
@@ -234,6 +237,8 @@ func getOrCreateFolderId(cfg *DashboardsAsConfig, service dashboards.DashboardPr
 		dash.Dashboard.IsFolder = true
 		dash.Overwrite = true
 		dash.OrgId = cfg.OrgId
+		// set dashboard folderUid if given
+		dash.Dashboard.SetUid(cfg.FolderUid)
 		dbDash, err := service.SaveFolderForProvisionedDashboards(dash)
 		if err != nil {
 			return 0, err

pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/dev-dashboards.yaml

@@ -4,6 +4,7 @@ providers:
 - name: 'general dashboards'
   orgId: 2
   folder: 'developers'
+  folderUid: 'xyz'
   editable: true
   disableDeletion: true
   updateIntervalSeconds: 15

pkg/services/provisioning/dashboards/testdata/test-configs/version-0/version-0.yaml

@@ -1,6 +1,7 @@
 - name: 'general dashboards'
   org_id: 2
   folder: 'developers'
+  folderUid: 'xyz'
   editable: true
   disableDeletion: true
   updateIntervalSeconds: 15

pkg/services/provisioning/dashboards/types.go

@@ -14,6 +14,7 @@ type DashboardsAsConfig struct {
 	Type                  string
 	OrgId                 int64
 	Folder                string
+	FolderUid             string
 	Editable              bool
 	Options               map[string]interface{}
 	DisableDeletion       bool
@@ -25,6 +26,7 @@ type DashboardsAsConfigV0 struct {
 	Type                  string                 `json:"type" yaml:"type"`
 	OrgId                 int64                  `json:"org_id" yaml:"org_id"`
 	Folder                string                 `json:"folder" yaml:"folder"`
+	FolderUid             string                 `json:"folderUid" yaml:"folderUid"`
 	Editable              bool                   `json:"editable" yaml:"editable"`
 	Options               map[string]interface{} `json:"options" yaml:"options"`
 	DisableDeletion       bool                   `json:"disableDeletion" yaml:"disableDeletion"`
@@ -44,6 +46,7 @@ type DashboardProviderConfigs struct {
 	Type                  string                 `json:"type" yaml:"type"`
 	OrgId                 int64                  `json:"orgId" yaml:"orgId"`
 	Folder                string                 `json:"folder" yaml:"folder"`
+	FolderUid             string                 `json:"folderUid" yaml:"folderUid"`
 	Editable              bool                   `json:"editable" yaml:"editable"`
 	Options               map[string]interface{} `json:"options" yaml:"options"`
 	DisableDeletion       bool                   `json:"disableDeletion" yaml:"disableDeletion"`
@@ -75,6 +78,7 @@ func mapV0ToDashboardAsConfig(v0 []*DashboardsAsConfigV0) []*DashboardsAsConfig
 			Type:                  v.Type,
 			OrgId:                 v.OrgId,
 			Folder:                v.Folder,
+			FolderUid:             v.FolderUid,
 			Editable:              v.Editable,
 			Options:               v.Options,
 			DisableDeletion:       v.DisableDeletion,
@@ -94,6 +98,7 @@ func (dc *DashboardAsConfigV1) mapToDashboardAsConfig() []*DashboardsAsConfig {
 			Type:                  v.Type,
 			OrgId:                 v.OrgId,
 			Folder:                v.Folder,
+			FolderUid:             v.FolderUid,
 			Editable:              v.Editable,
 			Options:               v.Options,
 			DisableDeletion:       v.DisableDeletion,