diff --git a/pkg/models/dashboards.go b/pkg/models/dashboards.go index 51df2e55a57..091f27ec413 100644 --- a/pkg/models/dashboards.go +++ b/pkg/models/dashboards.go @@ -139,8 +139,12 @@ func (dash *Dashboard) GetString(prop string, defaultValue string) string { // UpdateSlug updates the slug func (dash *Dashboard) UpdateSlug() { - title := strings.ToLower(dash.Data.Get("title").MustString()) - dash.Slug = slug.Make(title) + title := dash.Data.Get("title").MustString() + dash.Slug = SlugifyTitle(title) +} + +func SlugifyTitle(title string) string { + return slug.Make(strings.ToLower(title)) } // diff --git a/pkg/models/dashboards_test.go b/pkg/models/dashboards_test.go index 0ec773dfb97..ad865b575bb 100644 --- a/pkg/models/dashboards_test.go +++ b/pkg/models/dashboards_test.go @@ -16,6 +16,12 @@ func TestDashboardModel(t *testing.T) { So(dashboard.Slug, ShouldEqual, "grafana-play-home") }) + Convey("Can slugify title", t, func() { + slug := SlugifyTitle("Grafana Play Home") + + So(slug, ShouldEqual, "grafana-play-home") + }) + Convey("Given a dashboard json", t, func() { json := simplejson.New() json.Set("title", "test dash") diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 9c8120f6a8c..156c798ff76 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -2,6 +2,7 @@ package dashboards import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -15,21 +16,21 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" - gocache "github.com/patrickmn/go-cache" ) var ( checkDiskForChangesInterval time.Duration = time.Second * 3 + + ErrFolderNameMissing error = errors.New("Folder name missing") ) type fileReader struct { - Cfg *DashboardsAsConfig - Path string - FolderId int64 - log log.Logger - dashboardRepo dashboards.Repository - cache *gocache.Cache - createWalkFunc func(fr *fileReader) filepath.WalkFunc + Cfg *DashboardsAsConfig + Path string + log log.Logger + dashboardRepo dashboards.Repository + cache *DashboardCache + createWalk func(fr *fileReader, folderId int64) filepath.WalkFunc } func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) { @@ -43,19 +44,19 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade } return &fileReader{ - Cfg: cfg, - Path: path, - log: log, - dashboardRepo: dashboards.GetRepository(), - cache: gocache.New(5*time.Minute, 30*time.Minute), - createWalkFunc: createWalkFn, + Cfg: cfg, + Path: path, + log: log, + dashboardRepo: dashboards.GetRepository(), + cache: NewDashboardCache(), + createWalk: createWalkFn, }, nil } func (fr *fileReader) ReadAndListen(ctx context.Context) error { ticker := time.NewTicker(checkDiskForChangesInterval) - if err := fr.walkFolder(); err != nil { + if err := fr.startWalkingDisk(); err != nil { fr.log.Error("failed to search for dashboards", "error", err) } @@ -67,7 +68,9 @@ func (fr *fileReader) ReadAndListen(ctx context.Context) error { if !running { // avoid walking the filesystem in parallel. incase fs is very slow. running = true go func() { - fr.walkFolder() + if err := fr.startWalkingDisk(); err != nil { + fr.log.Error("failed to search for dashboards", "error", err) + } running = false }() } @@ -77,17 +80,56 @@ func (fr *fileReader) ReadAndListen(ctx context.Context) error { } } -func (fr *fileReader) walkFolder() error { +func (fr *fileReader) startWalkingDisk() error { if _, err := os.Stat(fr.Path); err != nil { if os.IsNotExist(err) { return err } } - return filepath.Walk(fr.Path, fr.createWalkFunc(fr)) //omg this is so ugly :( + folderId, err := getOrCreateFolder(fr.Cfg, fr.dashboardRepo) + if err != nil && err != ErrFolderNameMissing { + return err + } + + return filepath.Walk(fr.Path, fr.createWalk(fr, folderId)) } -func createWalkFn(fr *fileReader) filepath.WalkFunc { +func getOrCreateFolder(cfg *DashboardsAsConfig, repo dashboards.Repository) (int64, error) { + if cfg.Folder == "" { + return 0, ErrFolderNameMissing + } + + cmd := &models.GetDashboardQuery{Slug: models.SlugifyTitle(cfg.Folder), OrgId: cfg.OrgId} + err := bus.Dispatch(cmd) + + if err != nil && err != models.ErrDashboardNotFound { + return 0, err + } + + // dashboard folder not found. create one. + if err == models.ErrDashboardNotFound { + dash := &dashboards.SaveDashboardItem{} + dash.Dashboard = models.NewDashboard(cfg.Folder) + dash.Dashboard.IsFolder = true + dash.Overwrite = true + dash.OrgId = cfg.OrgId + dbDash, err := repo.SaveDashboard(dash) + if err != nil { + return 0, err + } + + return dbDash.Id, nil + } + + if !cmd.Result.IsFolder { + return 0, fmt.Errorf("Got invalid response. Expected folder, found dashboard") + } + + return cmd.Result.Id, nil +} + +func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc { return func(path string, fileInfo os.FileInfo, err error) error { if err != nil { return err @@ -103,12 +145,12 @@ func createWalkFn(fr *fileReader) filepath.WalkFunc { return nil } - cachedDashboard, exist := fr.getCache(path) + cachedDashboard, exist := fr.cache.getCache(path) if exist && cachedDashboard.UpdatedAt == fileInfo.ModTime() { return nil } - dash, err := fr.readDashboardFromFile(path) + dash, err := fr.readDashboardFromFile(path, folderId) if err != nil { fr.log.Error("failed to load dashboard from ", "file", path, "error", err) return nil @@ -143,7 +185,7 @@ func createWalkFn(fr *fileReader) filepath.WalkFunc { } } -func (fr *fileReader) readDashboardFromFile(path string) (*dashboards.SaveDashboardItem, error) { +func (fr *fileReader) readDashboardFromFile(path string, folderId int64) (*dashboards.SaveDashboardItem, error) { reader, err := os.Open(path) if err != nil { return nil, err @@ -160,30 +202,12 @@ func (fr *fileReader) readDashboardFromFile(path string) (*dashboards.SaveDashbo return nil, err } - dash, err := createDashboardJson(data, stat.ModTime(), fr.Cfg) + dash, err := createDashboardJson(data, stat.ModTime(), fr.Cfg, folderId) if err != nil { return nil, err } - fr.addDashboardCache(path, dash) + fr.cache.addDashboardCache(path, dash) return dash, nil } - -func (fr *fileReader) addDashboardCache(key string, json *dashboards.SaveDashboardItem) { - fr.cache.Add(key, json, time.Minute*10) -} - -func (fr *fileReader) getCache(key string) (*dashboards.SaveDashboardItem, bool) { - obj, exist := fr.cache.Get(key) - if !exist { - return nil, exist - } - - dash, ok := obj.(*dashboards.SaveDashboardItem) - if !ok { - return nil, ok - } - - return dash, ok -} diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index 84bdf6ee49d..b0cc6ee418f 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -24,15 +24,15 @@ var ( func TestDashboardFileReader(t *testing.T) { Convey("Dashboard file reader", t, func() { + bus.ClearBusHandlers() + fakeRepo = &fakeDashboardRepo{} + + bus.AddHandler("test", mockGetDashboardQuery) + dashboards.SetRepository(fakeRepo) + logger := log.New("test.logger") + Convey("Reading dashboards from disk", func() { - bus.ClearBusHandlers() - fakeRepo = &fakeDashboardRepo{} - - bus.AddHandler("test", mockGetDashboardQuery) - dashboards.SetRepository(fakeRepo) - logger := log.New("test.logger") - cfg := &DashboardsAsConfig{ Name: "Default", Type: "file", @@ -43,14 +43,27 @@ func TestDashboardFileReader(t *testing.T) { Convey("Can read default dashboard", func() { cfg.Options["folder"] = defaultDashboards + cfg.Folder = "Team A" reader, err := NewDashboardFileReader(cfg, logger) So(err, ShouldBeNil) - err = reader.walkFolder() + err = reader.startWalkingDisk() So(err, ShouldBeNil) - So(len(fakeRepo.inserted), ShouldEqual, 2) + folders := 0 + dashboards := 0 + + for _, i := range fakeRepo.inserted { + if i.Dashboard.IsFolder { + folders++ + } else { + dashboards++ + } + } + + So(dashboards, ShouldEqual, 2) + So(folders, ShouldEqual, 1) }) Convey("Should not update dashboards when db is newer", func() { @@ -64,7 +77,7 @@ func TestDashboardFileReader(t *testing.T) { reader, err := NewDashboardFileReader(cfg, logger) So(err, ShouldBeNil) - err = reader.walkFolder() + err = reader.startWalkingDisk() So(err, ShouldBeNil) So(len(fakeRepo.inserted), ShouldEqual, 0) @@ -83,7 +96,7 @@ func TestDashboardFileReader(t *testing.T) { reader, err := NewDashboardFileReader(cfg, logger) So(err, ShouldBeNil) - err = reader.walkFolder() + err = reader.startWalkingDisk() So(err, ShouldBeNil) So(len(fakeRepo.inserted), ShouldEqual, 1) @@ -102,22 +115,52 @@ func TestDashboardFileReader(t *testing.T) { }) Convey("Broken dashboards should not cause error", func() { - cfg := &DashboardsAsConfig{ - Name: "Default", - Type: "file", - OrgId: 1, - Folder: "", - Options: map[string]interface{}{ - "folder": brokenDashboards, - }, - } + cfg.Options["folder"] = brokenDashboards _, err := NewDashboardFileReader(cfg, logger) So(err, ShouldBeNil) }) }) - Convey("Walking", func() { + Convey("Should not create new folder if folder name is missing", func() { + cfg := &DashboardsAsConfig{ + Name: "Default", + Type: "file", + OrgId: 1, + Folder: "", + Options: map[string]interface{}{ + "folder": defaultDashboards, + }, + } + + _, err := getOrCreateFolder(cfg, fakeRepo) + So(err, ShouldEqual, ErrFolderNameMissing) + }) + + Convey("can get or Create dashboard folder", func() { + cfg := &DashboardsAsConfig{ + Name: "Default", + Type: "file", + OrgId: 1, + Folder: "TEAM A", + Options: map[string]interface{}{ + "folder": defaultDashboards, + }, + } + + folderId, err := getOrCreateFolder(cfg, fakeRepo) + So(err, ShouldBeNil) + inserted := false + for _, d := range fakeRepo.inserted { + if d.Dashboard.IsFolder && d.Dashboard.Id == folderId { + inserted = true + } + } + So(len(fakeRepo.inserted), ShouldEqual, 1) + So(inserted, ShouldBeTrue) + }) + + Convey("Walking the folder with dashboards", func() { cfg := &DashboardsAsConfig{ Name: "Default", Type: "file", @@ -132,12 +175,12 @@ func TestDashboardFileReader(t *testing.T) { So(err, ShouldBeNil) Convey("should skip dirs that starts with .", func() { - shouldSkip := reader.createWalkFunc(reader)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil) + shouldSkip := reader.createWalk(reader, 0)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil) So(shouldSkip, ShouldEqual, filepath.SkipDir) }) Convey("should keep walking if file is not .json", func() { - shouldSkip := reader.createWalkFunc(reader)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil) + shouldSkip := reader.createWalk(reader, 0)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil) So(shouldSkip, ShouldBeNil) }) }) diff --git a/pkg/services/provisioning/dashboards/types.go b/pkg/services/provisioning/dashboards/types.go index c4c4a67a755..4869da8927b 100644 --- a/pkg/services/provisioning/dashboards/types.go +++ b/pkg/services/provisioning/dashboards/types.go @@ -7,6 +7,7 @@ import ( "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/models" + gocache "github.com/patrickmn/go-cache" ) type DashboardsAsConfig struct { @@ -18,14 +19,42 @@ type DashboardsAsConfig struct { Options map[string]interface{} `json:"options" yaml:"options"` } -func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig) (*dashboards.SaveDashboardItem, error) { +type DashboardCache struct { + internalCache *gocache.Cache +} +func NewDashboardCache() *DashboardCache { + return &DashboardCache{internalCache: gocache.New(5*time.Minute, 30*time.Minute)} +} + +func (fr *DashboardCache) addDashboardCache(key string, json *dashboards.SaveDashboardItem) { + fr.internalCache.Add(key, json, time.Minute*10) +} + +func (fr *DashboardCache) getCache(key string) (*dashboards.SaveDashboardItem, bool) { + obj, exist := fr.internalCache.Get(key) + if !exist { + return nil, exist + } + + dash, ok := obj.(*dashboards.SaveDashboardItem) + if !ok { + return nil, ok + } + + return dash, ok +} + +func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig, folderId int64) (*dashboards.SaveDashboardItem, error) { dash := &dashboards.SaveDashboardItem{} dash.Dashboard = models.NewDashboardFromJson(data) dash.UpdatedAt = lastModified dash.Overwrite = true dash.OrgId = cfg.OrgId - dash.Dashboard.Data.Set("editable", cfg.Editable) + dash.Dashboard.FolderId = folderId + if !cfg.Editable { + dash.Dashboard.Data.Set("editable", cfg.Editable) + } if dash.Dashboard.Title == "" { return nil, models.ErrDashboardTitleEmpty