From ec9a587cbebfe3967deaf9e9300a7a4fbf08797b Mon Sep 17 00:00:00 2001 From: Maksim Nabokikh Date: Wed, 21 Jul 2021 19:52:41 +0400 Subject: [PATCH] Provisioning: Improve validation by validating across all dashboard providers (#26742) * Provisioning: check sanity across all dashboard readers Signed-off-by: m.nabokikh * Apply suggestions from code review Co-authored-by: Emil Tullstedt Co-authored-by: Arve Knudsen * Refactor of duplicateValidator and fix issues according to commentaries Signed-off-by: m.nabokikh * Apply suggestions from code review Co-authored-by: Arve Knudsen * Remove newDuplicateEntries function Signed-off-by: m.nabokikh * Change folderUid in logs to folderUID Signed-off-by: m.nabokikh * Restrict write access for readers, which are provisioning duplicate dashboards Signed-off-by: m.nabokikh * Fix file reader after rebasing onto master Signed-off-by: m.nabokikh * Apply suggestions from code review Co-authored-by: Arve Knudsen * Format file_reader Signed-off-by: m.nabokikh * Apply suggestions from code review Co-authored-by: Arve Knudsen * Apply suggestions from code review Co-authored-by: Arve Knudsen * Fix lint problem Signed-off-by: m.nabokikh * Apply suggestions from code review Co-authored-by: Marcus Efraimsson Signed-off-by: m.nabokikh Co-authored-by: Emil Tullstedt Co-authored-by: Arve Knudsen Co-authored-by: Marcus Efraimsson --- .../provisioning/dashboards/dashboard.go | 17 +- .../provisioning/dashboards/file_reader.go | 107 ++++++----- .../dashboards/file_reader_linux_test.go | 4 +- .../dashboard-with-uid/dashboard1.json | 173 ++++++++++++++++++ .../two-dashboards-with-uid/dashboard1.json | 173 ++++++++++++++++++ .../two-dashboards-with-uid/dashboard2.json | 173 ++++++++++++++++++ .../provisioning/dashboards/validator.go | 145 +++++++++++++++ .../provisioning/dashboards/validator_test.go | 124 +++++++++++++ 8 files changed, 865 insertions(+), 51 deletions(-) create mode 100644 pkg/services/provisioning/dashboards/testdata/test-dashboards/dashboard-with-uid/dashboard1.json create mode 100644 pkg/services/provisioning/dashboards/testdata/test-dashboards/two-dashboards-with-uid/dashboard1.json create mode 100644 pkg/services/provisioning/dashboards/testdata/test-dashboards/two-dashboards-with-uid/dashboard2.json create mode 100644 pkg/services/provisioning/dashboards/validator.go create mode 100644 pkg/services/provisioning/dashboards/validator_test.go diff --git a/pkg/services/provisioning/dashboards/dashboard.go b/pkg/services/provisioning/dashboards/dashboard.go index 4847c6af884..bafd171d833 100644 --- a/pkg/services/provisioning/dashboards/dashboard.go +++ b/pkg/services/provisioning/dashboards/dashboard.go @@ -27,9 +27,10 @@ type DashboardProvisionerFactory func(string, dashboards.Store) (DashboardProvis // Provisioner is responsible for syncing dashboard from disk to Grafana's database. type Provisioner struct { - log log.Logger - fileReaders []*FileReader - configs []*config + log log.Logger + fileReaders []*FileReader + configs []*config + duplicateValidator duplicateValidator } // New returns a new DashboardProvisioner @@ -47,9 +48,10 @@ func New(configDirectory string, store dashboards.Store) (DashboardProvisioner, } d := &Provisioner{ - log: logger, - fileReaders: fileReaders, - configs: configs, + log: logger, + fileReaders: fileReaders, + configs: configs, + duplicateValidator: newDuplicateValidator(logger, fileReaders), } return d, nil @@ -70,6 +72,7 @@ func (provider *Provisioner) Provision() error { } } + provider.duplicateValidator.validate() return nil } @@ -92,6 +95,8 @@ func (provider *Provisioner) PollChanges(ctx context.Context) { for _, reader := range provider.fileReaders { go reader.pollChanges(ctx) } + + go provider.duplicateValidator.Run(ctx) } // GetProvisionerResolvedPath returns resolved path for the specified provisioner name. Can be used to generate diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 99c835bd319..32ac06be608 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "sync" "time" "github.com/grafana/grafana/pkg/bus" @@ -33,6 +34,10 @@ type FileReader struct { log log.Logger dashboardProvisioningService dashboards.DashboardProvisioningService FoldersFromFilesStructure bool + + mux sync.RWMutex + usageTracker *usageTracker + dbWriteAccessRestricted bool } // NewDashboardFileReader returns a new filereader based on `config` @@ -59,6 +64,7 @@ func NewDashboardFileReader(cfg *config, log log.Logger, store dboards.Store) (* log: log, dashboardProvisioningService: dashboards.NewProvisioningService(store), FoldersFromFilesStructure: foldersFromFilesStructure, + usageTracker: newUsageTracker(), }, nil } @@ -99,25 +105,40 @@ func (fr *FileReader) walkDisk() error { fr.handleMissingDashboardFiles(provisionedDashboardRefs, filesFoundOnDisk) - sanityChecker := newProvisioningSanityChecker(fr.Cfg.Name) - + usageTracker := newUsageTracker() if fr.FoldersFromFilesStructure { - err = fr.storeDashboardsInFoldersFromFileStructure(filesFoundOnDisk, provisionedDashboardRefs, resolvedPath, &sanityChecker) + err = fr.storeDashboardsInFoldersFromFileStructure(filesFoundOnDisk, provisionedDashboardRefs, resolvedPath, usageTracker) } else { - err = fr.storeDashboardsInFolder(filesFoundOnDisk, provisionedDashboardRefs, &sanityChecker) + err = fr.storeDashboardsInFolder(filesFoundOnDisk, provisionedDashboardRefs, usageTracker) } if err != nil { return err } - sanityChecker.logWarnings(fr.log) + fr.mux.Lock() + defer fr.mux.Unlock() + fr.usageTracker = usageTracker return nil } +func (fr *FileReader) changeWritePermissions(restrict bool) { + fr.mux.Lock() + defer fr.mux.Unlock() + + fr.dbWriteAccessRestricted = restrict +} + +func (fr *FileReader) isDatabaseAccessRestricted() bool { + fr.mux.RLock() + defer fr.mux.RUnlock() + + return fr.dbWriteAccessRestricted +} + // storeDashboardsInFolder saves dashboards from the filesystem on disk to the folder from config func (fr *FileReader) storeDashboardsInFolder(filesFoundOnDisk map[string]os.FileInfo, - dashboardRefs map[string]*models.DashboardProvisioning, sanityChecker *provisioningSanityChecker) error { + dashboardRefs map[string]*models.DashboardProvisioning, usageTracker *usageTracker) error { folderID, err := getOrCreateFolderID(fr.Cfg, fr.dashboardProvisioningService, fr.Cfg.Folder) if err != nil && !errors.Is(err, ErrFolderNameMissing) { return err @@ -131,7 +152,7 @@ func (fr *FileReader) storeDashboardsInFolder(filesFoundOnDisk map[string]os.Fil continue } - sanityChecker.track(provisioningMetadata) + usageTracker.track(provisioningMetadata) } return nil } @@ -139,7 +160,7 @@ func (fr *FileReader) storeDashboardsInFolder(filesFoundOnDisk map[string]os.Fil // storeDashboardsInFoldersFromFilesystemStructure saves dashboards from the filesystem on disk to the same folder // in Grafana as they are in on the filesystem. func (fr *FileReader) storeDashboardsInFoldersFromFileStructure(filesFoundOnDisk map[string]os.FileInfo, - dashboardRefs map[string]*models.DashboardProvisioning, resolvedPath string, sanityChecker *provisioningSanityChecker) error { + dashboardRefs map[string]*models.DashboardProvisioning, resolvedPath string, usageTracker *usageTracker) error { for path, fileInfo := range filesFoundOnDisk { folderName := "" @@ -154,7 +175,7 @@ func (fr *FileReader) storeDashboardsInFoldersFromFileStructure(filesFoundOnDisk } provisioningMetadata, err := fr.saveDashboard(path, folderID, fileInfo, dashboardRefs) - sanityChecker.track(provisioningMetadata) + usageTracker.track(provisioningMetadata) if err != nil { fr.log.Error("failed to save dashboard", "error", err) } @@ -236,16 +257,23 @@ func (fr *FileReader) saveDashboard(path string, folderID int64, fileInfo os.Fil dash.Dashboard.SetId(provisionedData.DashboardId) } - fr.log.Debug("saving new dashboard", "provisioner", fr.Cfg.Name, "file", path, "folderId", dash.Dashboard.FolderId) - dp := &models.DashboardProvisioning{ - ExternalId: path, - Name: fr.Cfg.Name, - Updated: resolvedFileInfo.ModTime().Unix(), - CheckSum: jsonFile.checkSum, + if !fr.isDatabaseAccessRestricted() { + fr.log.Debug("saving new dashboard", "provisioner", fr.Cfg.Name, "file", path, "folderId", dash.Dashboard.FolderId) + dp := &models.DashboardProvisioning{ + ExternalId: path, + Name: fr.Cfg.Name, + Updated: resolvedFileInfo.ModTime().Unix(), + CheckSum: jsonFile.checkSum, + } + if _, err := fr.dashboardProvisioningService.SaveProvisionedDashboard(dash, dp); err != nil { + return provisioningMetadata, err + } + } else { + fr.log.Warn("Not saving new dashboard due to restricted database access", "provisioner", fr.Cfg.Name, + "file", path, "folderId", dash.Dashboard.FolderId) } - _, err = fr.dashboardProvisioningService.SaveProvisionedDashboard(dash, dp) - return provisioningMetadata, err + return provisioningMetadata, nil } func getProvisionedDashboardsByPath(service dashboards.DashboardProvisioningService, name string) ( @@ -412,6 +440,13 @@ func (fr *FileReader) resolvedPath() string { return path } +func (fr *FileReader) getUsageTracker() *usageTracker { + fr.mux.RLock() + defer fr.mux.RUnlock() + + return fr.usageTracker +} + type provisioningMetadata struct { uid string identity dashboardIdentity @@ -423,42 +458,26 @@ type dashboardIdentity struct { } func (d *dashboardIdentity) Exists() bool { - return len(d.title) > 0 && d.folderID > 0 + return len(d.title) > 0 } -func newProvisioningSanityChecker(provisioningProvider string) provisioningSanityChecker { - return provisioningSanityChecker{ - provisioningProvider: provisioningProvider, - uidUsage: map[string]uint8{}, - titleUsage: map[dashboardIdentity]uint8{}, +func newUsageTracker() *usageTracker { + return &usageTracker{ + uidUsage: map[string]uint8{}, + titleUsage: map[dashboardIdentity]uint8{}, } } -type provisioningSanityChecker struct { - provisioningProvider string - uidUsage map[string]uint8 - titleUsage map[dashboardIdentity]uint8 +type usageTracker struct { + uidUsage map[string]uint8 + titleUsage map[dashboardIdentity]uint8 } -func (checker provisioningSanityChecker) track(pm provisioningMetadata) { +func (t *usageTracker) track(pm provisioningMetadata) { if len(pm.uid) > 0 { - checker.uidUsage[pm.uid]++ + t.uidUsage[pm.uid]++ } if pm.identity.Exists() { - checker.titleUsage[pm.identity]++ - } -} - -func (checker provisioningSanityChecker) logWarnings(log log.Logger) { - for uid, times := range checker.uidUsage { - if times > 1 { - log.Error("the same 'uid' is used more than once", "uid", uid, "provider", checker.provisioningProvider) - } - } - - for identity, times := range checker.titleUsage { - if times > 1 { - log.Error("the same 'title' is used more than once", "title", identity.title, "provider", checker.provisioningProvider) - } + t.titleUsage[pm.identity]++ } } diff --git a/pkg/services/provisioning/dashboards/file_reader_linux_test.go b/pkg/services/provisioning/dashboards/file_reader_linux_test.go index b32c642551b..5ccddddf810 100644 --- a/pkg/services/provisioning/dashboards/file_reader_linux_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_linux_test.go @@ -25,7 +25,9 @@ func TestProvisionedSymlinkedFolder(t *testing.T) { } reader, err := NewDashboardFileReader(cfg, log.New("test-logger"), nil) - require.NoError(t, err) + if err != nil { + t.Error("expected err to be nil") + } want, err := filepath.Abs(containingID) require.NoError(t, err) diff --git a/pkg/services/provisioning/dashboards/testdata/test-dashboards/dashboard-with-uid/dashboard1.json b/pkg/services/provisioning/dashboards/testdata/test-dashboards/dashboard-with-uid/dashboard1.json new file mode 100644 index 00000000000..5527ecf5ff0 --- /dev/null +++ b/pkg/services/provisioning/dashboards/testdata/test-dashboards/dashboard-with-uid/dashboard1.json @@ -0,0 +1,173 @@ +{ + "title": "Grafana", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "rows": [ + { + "title": "New row", + "height": "150px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 1, + "span": 12, + "editable": true, + "type": "text", + "mode": "html", + "content": "
\n \n
", + "style": {}, + "title": "Welcome to" + } + ] + }, + { + "title": "Welcome to Grafana", + "height": "210px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 2, + "span": 6, + "type": "text", + "mode": "html", + "content": "
\n\n
\n
\n \n
\n
\n \n
\n
", + "style": {}, + "title": "Documentation Links" + }, + { + "id": 3, + "span": 6, + "type": "text", + "mode": "html", + "content": "
\n\n
\n
\n
    \n
  • Ctrl+S saves the current dashboard
    • \n
  • Ctrl+F Opens the dashboard finder
    • \n
  • Ctrl+H Hide/show row controls
    • \n
  • Click and drag graph title to move panel
    • \n
  • Hit Escape to exit graph when in fullscreen or edit mode
    • \n
  • Click the colored icon in the legend to change series color
    • \n
  • Ctrl or Shift + Click legend name to hide other series
    • \n
\n
\n
\n", + "style": {}, + "title": "Tips & Shortcuts" + } + ] + }, + { + "title": "test", + "height": "250px", + "editable": true, + "collapse": false, + "panels": [ + { + "id": 4, + "span": 12, + "type": "graph", + "x-axis": true, + "y-axis": true, + "scale": 1, + "y_formats": [ + "short", + "short" + ], + "grid": { + "max": null, + "min": null, + "leftMax": null, + "rightMax": null, + "leftMin": null, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "resolution": 100, + "lines": true, + "fill": 1, + "linewidth": 2, + "dashes": false, + "dashLength": 10, + "spaceLength": 10, + "points": false, + "pointradius": 5, + "bars": false, + "stack": true, + "spyable": true, + "options": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "interactive": true, + "legend_counts": true, + "timezone": "browser", + "percentage": false, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": true + }, + "targets": [ + { + "target": "randomWalk('random walk')", + "function": "mean", + "column": "value" + } + ], + "aliasColors": {}, + "aliasYAxis": {}, + "title": "First Graph (click title to edit)", + "datasource": "graphite", + "renderer": "flot", + "annotate": { + "enable": false + } + } + ] + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "enable": true, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "now": true + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [] + }, + "version": 5, + "uid": "Z-phNqGmz" +} diff --git a/pkg/services/provisioning/dashboards/testdata/test-dashboards/two-dashboards-with-uid/dashboard1.json b/pkg/services/provisioning/dashboards/testdata/test-dashboards/two-dashboards-with-uid/dashboard1.json new file mode 100644 index 00000000000..5527ecf5ff0 --- /dev/null +++ b/pkg/services/provisioning/dashboards/testdata/test-dashboards/two-dashboards-with-uid/dashboard1.json @@ -0,0 +1,173 @@ +{ + "title": "Grafana", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "rows": [ + { + "title": "New row", + "height": "150px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 1, + "span": 12, + "editable": true, + "type": "text", + "mode": "html", + "content": "
\n \n
", + "style": {}, + "title": "Welcome to" + } + ] + }, + { + "title": "Welcome to Grafana", + "height": "210px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 2, + "span": 6, + "type": "text", + "mode": "html", + "content": "
\n\n
\n
\n \n
\n
\n \n
\n
", + "style": {}, + "title": "Documentation Links" + }, + { + "id": 3, + "span": 6, + "type": "text", + "mode": "html", + "content": "
\n\n
\n
\n
    \n
  • Ctrl+S saves the current dashboard
    • \n
  • Ctrl+F Opens the dashboard finder
    • \n
  • Ctrl+H Hide/show row controls
    • \n
  • Click and drag graph title to move panel
    • \n
  • Hit Escape to exit graph when in fullscreen or edit mode
    • \n
  • Click the colored icon in the legend to change series color
    • \n
  • Ctrl or Shift + Click legend name to hide other series
    • \n
\n
\n
\n", + "style": {}, + "title": "Tips & Shortcuts" + } + ] + }, + { + "title": "test", + "height": "250px", + "editable": true, + "collapse": false, + "panels": [ + { + "id": 4, + "span": 12, + "type": "graph", + "x-axis": true, + "y-axis": true, + "scale": 1, + "y_formats": [ + "short", + "short" + ], + "grid": { + "max": null, + "min": null, + "leftMax": null, + "rightMax": null, + "leftMin": null, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "resolution": 100, + "lines": true, + "fill": 1, + "linewidth": 2, + "dashes": false, + "dashLength": 10, + "spaceLength": 10, + "points": false, + "pointradius": 5, + "bars": false, + "stack": true, + "spyable": true, + "options": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "interactive": true, + "legend_counts": true, + "timezone": "browser", + "percentage": false, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": true + }, + "targets": [ + { + "target": "randomWalk('random walk')", + "function": "mean", + "column": "value" + } + ], + "aliasColors": {}, + "aliasYAxis": {}, + "title": "First Graph (click title to edit)", + "datasource": "graphite", + "renderer": "flot", + "annotate": { + "enable": false + } + } + ] + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "enable": true, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "now": true + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [] + }, + "version": 5, + "uid": "Z-phNqGmz" +} diff --git a/pkg/services/provisioning/dashboards/testdata/test-dashboards/two-dashboards-with-uid/dashboard2.json b/pkg/services/provisioning/dashboards/testdata/test-dashboards/two-dashboards-with-uid/dashboard2.json new file mode 100644 index 00000000000..5527ecf5ff0 --- /dev/null +++ b/pkg/services/provisioning/dashboards/testdata/test-dashboards/two-dashboards-with-uid/dashboard2.json @@ -0,0 +1,173 @@ +{ + "title": "Grafana", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "rows": [ + { + "title": "New row", + "height": "150px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 1, + "span": 12, + "editable": true, + "type": "text", + "mode": "html", + "content": "
\n \n
", + "style": {}, + "title": "Welcome to" + } + ] + }, + { + "title": "Welcome to Grafana", + "height": "210px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 2, + "span": 6, + "type": "text", + "mode": "html", + "content": "
\n\n
\n
\n \n
\n
\n \n
\n
", + "style": {}, + "title": "Documentation Links" + }, + { + "id": 3, + "span": 6, + "type": "text", + "mode": "html", + "content": "
\n\n
\n
\n
    \n
  • Ctrl+S saves the current dashboard
    • \n
  • Ctrl+F Opens the dashboard finder
    • \n
  • Ctrl+H Hide/show row controls
    • \n
  • Click and drag graph title to move panel
    • \n
  • Hit Escape to exit graph when in fullscreen or edit mode
    • \n
  • Click the colored icon in the legend to change series color
    • \n
  • Ctrl or Shift + Click legend name to hide other series
    • \n
\n
\n
\n", + "style": {}, + "title": "Tips & Shortcuts" + } + ] + }, + { + "title": "test", + "height": "250px", + "editable": true, + "collapse": false, + "panels": [ + { + "id": 4, + "span": 12, + "type": "graph", + "x-axis": true, + "y-axis": true, + "scale": 1, + "y_formats": [ + "short", + "short" + ], + "grid": { + "max": null, + "min": null, + "leftMax": null, + "rightMax": null, + "leftMin": null, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "resolution": 100, + "lines": true, + "fill": 1, + "linewidth": 2, + "dashes": false, + "dashLength": 10, + "spaceLength": 10, + "points": false, + "pointradius": 5, + "bars": false, + "stack": true, + "spyable": true, + "options": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "interactive": true, + "legend_counts": true, + "timezone": "browser", + "percentage": false, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": true + }, + "targets": [ + { + "target": "randomWalk('random walk')", + "function": "mean", + "column": "value" + } + ], + "aliasColors": {}, + "aliasYAxis": {}, + "title": "First Graph (click title to edit)", + "datasource": "graphite", + "renderer": "flot", + "annotate": { + "enable": false + } + } + ] + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "enable": true, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "now": true + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [] + }, + "version": 5, + "uid": "Z-phNqGmz" +} diff --git a/pkg/services/provisioning/dashboards/validator.go b/pkg/services/provisioning/dashboards/validator.go new file mode 100644 index 00000000000..e5402b28382 --- /dev/null +++ b/pkg/services/provisioning/dashboards/validator.go @@ -0,0 +1,145 @@ +package dashboards + +import ( + "context" + "time" + + "github.com/grafana/grafana/pkg/infra/log" +) + +type duplicate struct { + Sum uint8 + InvolvedReaders map[string]struct{} +} + +func newDuplicate() *duplicate { + return &duplicate{InvolvedReaders: make(map[string]struct{})} +} + +type duplicateEntries struct { + Titles map[dashboardIdentity]*duplicate + UIDs map[string]*duplicate +} + +func (d *duplicateEntries) InvolvedReaders() map[string]struct{} { + involvedReaders := make(map[string]struct{}) + + for _, duplicate := range d.UIDs { + if duplicate.Sum <= 1 { + continue + } + + for readerName := range duplicate.InvolvedReaders { + involvedReaders[readerName] = struct{}{} + } + } + + for _, duplicate := range d.Titles { + if duplicate.Sum <= 1 { + continue + } + + for readerName := range duplicate.InvolvedReaders { + involvedReaders[readerName] = struct{}{} + } + } + + return involvedReaders +} + +type duplicateValidator struct { + logger log.Logger + readers []*FileReader +} + +func newDuplicateValidator(logger log.Logger, readers []*FileReader) duplicateValidator { + return duplicateValidator{logger: logger, readers: readers} +} + +func (c *duplicateValidator) getDuplicates() *duplicateEntries { + duplicates := duplicateEntries{ + Titles: make(map[dashboardIdentity]*duplicate), + UIDs: make(map[string]*duplicate), + } + + for _, reader := range c.readers { + readerName := reader.Cfg.Name + tracker := reader.getUsageTracker() + + for uid, times := range tracker.uidUsage { + if _, ok := duplicates.UIDs[uid]; !ok { + duplicates.UIDs[uid] = newDuplicate() + } + duplicates.UIDs[uid].Sum += times + duplicates.UIDs[uid].InvolvedReaders[readerName] = struct{}{} + } + + for id, times := range tracker.titleUsage { + if _, ok := duplicates.Titles[id]; !ok { + duplicates.Titles[id] = newDuplicate() + } + duplicates.Titles[id].Sum += times + duplicates.Titles[id].InvolvedReaders[readerName] = struct{}{} + } + } + + return &duplicates +} + +func (c *duplicateValidator) logWarnings(duplicates *duplicateEntries) { + for uid, usage := range duplicates.UIDs { + if usage.Sum > 1 { + c.logger.Warn("the same UID is used more than once", "uid", uid, "times", usage.Sum, "providers", + keysToSlice(usage.InvolvedReaders)) + } + } + + for id, usage := range duplicates.Titles { + if usage.Sum > 1 { + c.logger.Warn("dashboard title is not unique in folder", "title", id.title, "folderID", id.folderID, "times", + usage.Sum, "providers", keysToSlice(usage.InvolvedReaders)) + } + } +} + +func (c *duplicateValidator) takeAwayWritePermissions(duplicates *duplicateEntries) { + involvedReaders := duplicates.InvolvedReaders() + for _, reader := range c.readers { + _, isReaderWithDuplicates := involvedReaders[reader.Cfg.Name] + // We restrict reader permissions to write to the database here to prevent overloading + reader.changeWritePermissions(isReaderWithDuplicates) + + if isReaderWithDuplicates { + c.logger.Warn("dashboards provisioning provider has no database write permissions because of duplicates", "provider", reader.Cfg.Name) + } + } +} + +func (c *duplicateValidator) Run(ctx context.Context) { + ticker := time.NewTicker(30 * time.Second) + for { + select { + case <-ticker.C: + c.validate() + case <-ctx.Done(): + return + } + } +} + +func (c *duplicateValidator) validate() { + duplicates := c.getDuplicates() + + c.logWarnings(duplicates) + c.takeAwayWritePermissions(duplicates) +} + +func keysToSlice(data map[string]struct{}) []string { + entries := make([]string, 0, len(data)) + + for entry := range data { + entries = append(entries, entry) + } + + return entries +} diff --git a/pkg/services/provisioning/dashboards/validator_test.go b/pkg/services/provisioning/dashboards/validator_test.go new file mode 100644 index 00000000000..a170f82cf30 --- /dev/null +++ b/pkg/services/provisioning/dashboards/validator_test.go @@ -0,0 +1,124 @@ +package dashboards + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/infra/log" +) + +const ( + dashboardContainingUID = "testdata/test-dashboards/dashboard-with-uid" + twoDashboardsWithUID = "testdata/test-dashboards/two-dashboards-with-uid" +) + +func TestDuplicatesValidator(t *testing.T) { + bus.ClearBusHandlers() + fakeService = mockDashboardProvisioningService() + + bus.AddHandler("test", mockGetDashboardQuery) + cfg := &config{ + Name: "Default", + Type: "file", + OrgID: 1, + Folder: "", + Options: map[string]interface{}{}, + } + logger := log.New("test.logger") + + t.Run("Duplicates validator should collect info about duplicate UIDs and titles within folders", func(t *testing.T) { + const folderName = "duplicates-validator-folder" + folderID, err := getOrCreateFolderID(cfg, fakeService, folderName) + require.NoError(t, err) + + identity := dashboardIdentity{folderID: folderID, title: "Grafana"} + + cfg1 := &config{ + Name: "first", Type: "file", OrgID: 1, Folder: folderName, + Options: map[string]interface{}{"path": dashboardContainingUID}, + } + cfg2 := &config{ + Name: "second", Type: "file", OrgID: 1, Folder: folderName, + Options: map[string]interface{}{"path": dashboardContainingUID}, + } + + reader1, err := NewDashboardFileReader(cfg1, logger, nil) + require.NoError(t, err) + + reader2, err := NewDashboardFileReader(cfg2, logger, nil) + require.NoError(t, err) + + duplicateValidator := newDuplicateValidator(logger, []*FileReader{reader1, reader2}) + + err = reader1.walkDisk() + require.NoError(t, err) + + err = reader2.walkDisk() + require.NoError(t, err) + + duplicates := duplicateValidator.getDuplicates() + + require.Equal(t, uint8(2), duplicates.UIDs["Z-phNqGmz"].Sum) + uidUsageReaders := keysToSlice(duplicates.UIDs["Z-phNqGmz"].InvolvedReaders) + sort.Strings(uidUsageReaders) + require.Equal(t, []string{"first", "second"}, uidUsageReaders) + + require.Equal(t, uint8(2), duplicates.Titles[identity].Sum) + titleUsageReaders := keysToSlice(duplicates.Titles[identity].InvolvedReaders) + sort.Strings(titleUsageReaders) + require.Equal(t, []string{"first", "second"}, titleUsageReaders) + + duplicateValidator.validate() + require.True(t, reader1.isDatabaseAccessRestricted()) + require.True(t, reader2.isDatabaseAccessRestricted()) + }) + + t.Run("Duplicates validator should restrict write access only for readers with duplicates", func(t *testing.T) { + cfg1 := &config{ + Name: "first", Type: "file", OrgID: 1, Folder: "duplicates-validator-folder", + Options: map[string]interface{}{"path": twoDashboardsWithUID}, + } + cfg2 := &config{ + Name: "second", Type: "file", OrgID: 1, Folder: "root", + Options: map[string]interface{}{"path": defaultDashboards}, + } + + reader1, err := NewDashboardFileReader(cfg1, logger, nil) + require.NoError(t, err) + + reader2, err := NewDashboardFileReader(cfg2, logger, nil) + require.NoError(t, err) + + duplicateValidator := newDuplicateValidator(logger, []*FileReader{reader1, reader2}) + + err = reader1.walkDisk() + require.NoError(t, err) + + err = reader2.walkDisk() + require.NoError(t, err) + + duplicates := duplicateValidator.getDuplicates() + + folderID, err := getOrCreateFolderID(cfg, fakeService, cfg1.Folder) + require.NoError(t, err) + + identity := dashboardIdentity{folderID: folderID, title: "Grafana"} + + require.Equal(t, uint8(2), duplicates.UIDs["Z-phNqGmz"].Sum) + uidUsageReaders := keysToSlice(duplicates.UIDs["Z-phNqGmz"].InvolvedReaders) + sort.Strings(uidUsageReaders) + require.Equal(t, []string{"first"}, uidUsageReaders) + + require.Equal(t, uint8(2), duplicates.Titles[identity].Sum) + titleUsageReaders := keysToSlice(duplicates.Titles[identity].InvolvedReaders) + sort.Strings(titleUsageReaders) + require.Equal(t, []string{"first"}, titleUsageReaders) + + duplicateValidator.validate() + require.True(t, reader1.isDatabaseAccessRestricted()) + require.False(t, reader2.isDatabaseAccessRestricted()) + }) +}