From a40299b4dc8c891293829925f606b164b5595051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 12 May 2015 12:20:03 +0200 Subject: [PATCH] Progress on json file index and search #960 --- pkg/api/search.go | 1 + pkg/models/dashboards.go | 14 ++- pkg/services/search/json_index.go | 114 +++++++++++++++++++++++++ pkg/services/search/json_index_test.go | 35 ++++++++ pkg/services/search/jsonfiles.go | 12 --- pkg/services/search/search.go | 36 +++++++- pkg/setting/setting.go | 6 +- 7 files changed, 197 insertions(+), 21 deletions(-) create mode 100644 pkg/services/search/json_index.go create mode 100644 pkg/services/search/json_index_test.go delete mode 100644 pkg/services/search/jsonfiles.go diff --git a/pkg/api/search.go b/pkg/api/search.go index 592377129a2..588708e2de9 100644 --- a/pkg/api/search.go +++ b/pkg/api/search.go @@ -35,6 +35,7 @@ func Search(c *middleware.Context) { result.TagsOnly = true } else { + query := search.Query{ Title: query, Tag: tag, diff --git a/pkg/models/dashboards.go b/pkg/models/dashboards.go index 234ae74f04a..3a023715b0c 100644 --- a/pkg/models/dashboards.go +++ b/pkg/models/dashboards.go @@ -54,12 +54,10 @@ func (dash *Dashboard) GetTags() []string { return b } -// GetDashboardModel turns the command into the savable model -func (cmd *SaveDashboardCommand) GetDashboardModel() *Dashboard { +func NewDashboardFromJson(data map[string]interface{}) *Dashboard { dash := &Dashboard{} - dash.Data = cmd.Dashboard + dash.Data = data dash.Title = dash.Data["title"].(string) - dash.OrgId = cmd.OrgId dash.UpdateSlug() if dash.Data["id"] != nil { @@ -75,6 +73,14 @@ func (cmd *SaveDashboardCommand) GetDashboardModel() *Dashboard { return dash } +// GetDashboardModel turns the command into the savable model +func (cmd *SaveDashboardCommand) GetDashboardModel() *Dashboard { + dash := NewDashboardFromJson(cmd.Dashboard) + dash.OrgId = cmd.OrgId + dash.UpdateSlug() + return dash +} + // GetString a func (dash *Dashboard) GetString(prop string) string { return dash.Data[prop].(string) diff --git a/pkg/services/search/json_index.go b/pkg/services/search/json_index.go new file mode 100644 index 00000000000..4ad63157a5e --- /dev/null +++ b/pkg/services/search/json_index.go @@ -0,0 +1,114 @@ +package search + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + + "github.com/grafana/grafana/pkg/log" + m "github.com/grafana/grafana/pkg/models" +) + +type JsonDashIndex struct { + path string + orgsIds []int64 + items []*JsonDashIndexItem +} + +type JsonDashIndexItem struct { + Title string + TitleLower string + Tags []string + TagsCsv string + FilePath string +} + +func NewJsonDashIndex(path string, orgIds string) *JsonDashIndex { + index := JsonDashIndex{} + index.path = path + // if orgIds != "" || orgIds != "*" { + // } + + index.updateIndex() + return &index +} + +func (index *JsonDashIndex) Search(query *Query) ([]*m.DashboardSearchHit, error) { + results := make([]*m.DashboardSearchHit, 0) + + for _, item := range index.items { + if strings.Contains(item.TitleLower, query.Title) { + results = append(results, &m.DashboardSearchHit{ + Title: item.Title, + Tags: item.Tags, + Slug: item.FilePath, + }) + } + } + + return results, nil +} + +func (index *JsonDashIndex) updateIndex() error { + log.Info("Updating JSON dashboard index, path: %v", index.path) + + index.items = make([]*JsonDashIndexItem, 0) + + visitor := func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + if f.IsDir() { + return nil + } + if strings.HasSuffix(f.Name(), ".json") { + err = index.loadDashboardIntoCache(path) + if err != nil { + return err + } + } + return nil + } + + if err := filepath.Walk(index.path, visitor); err != nil { + return err + } + + return nil +} + +func (index *JsonDashIndex) loadDashboardIntoCache(filename string) error { + dash, err := loadDashboardFromFile(filename) + if err != nil { + return err + } + + index.items = append(index.items, dash) + return nil +} + +func loadDashboardFromFile(filename string) (*JsonDashIndexItem, error) { + reader, err := os.Open(filename) + if err != nil { + return nil, err + } + defer reader.Close() + + jsonParser := json.NewDecoder(reader) + var data map[string]interface{} + + if err := jsonParser.Decode(&data); err != nil { + return nil, err + } + + dash := m.NewDashboardFromJson(data) + + item := &JsonDashIndexItem{} + item.Title = dash.Title + item.TitleLower = strings.ToLower(item.Title) + item.Tags = dash.GetTags() + item.TagsCsv = strings.Join(item.Tags, ",") + + return item, nil +} diff --git a/pkg/services/search/json_index_test.go b/pkg/services/search/json_index_test.go new file mode 100644 index 00000000000..ddae1f20ace --- /dev/null +++ b/pkg/services/search/json_index_test.go @@ -0,0 +1,35 @@ +package search + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestJsonDashIndex(t *testing.T) { + + Convey("Given the json dash index", t, func() { + index := NewJsonDashIndex("../../../public/dashboards/", "*") + + Convey("Should be able to update index", func() { + err := index.updateIndex() + So(err, ShouldBeNil) + }) + + Convey("Should be able to search index", func() { + res, err := index.Search(&Query{Title: "", Tag: ""}) + So(err, ShouldBeNil) + + So(len(res), ShouldEqual, 4) + }) + + Convey("Should be able to search index by title", func() { + res, err := index.Search(&Query{Title: "home", Tag: ""}) + So(err, ShouldBeNil) + + So(len(res), ShouldEqual, 1) + So(res[0].Title, ShouldEqual, "Home") + }) + + }) +} diff --git a/pkg/services/search/jsonfiles.go b/pkg/services/search/jsonfiles.go deleted file mode 100644 index 0ddcf62e6c8..00000000000 --- a/pkg/services/search/jsonfiles.go +++ /dev/null @@ -1,12 +0,0 @@ -package search - -var ( - // settings - DashboardsJsonEnabled bool - DashboardsJsonPath string - DashboardJsonOrgs string -) - -func initJsonFileIndex() { - -} diff --git a/pkg/services/search/search.go b/pkg/services/search/search.go index 93d49ec85a3..e32f36908c5 100644 --- a/pkg/services/search/search.go +++ b/pkg/services/search/search.go @@ -1,8 +1,11 @@ package search import ( + "path/filepath" + "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" ) type Query struct { @@ -16,12 +19,28 @@ type Query struct { Result []*m.DashboardSearchHit } +var jsonDashIndex *JsonDashIndex + func Init() { bus.AddHandler("search", searchHandler) - initJsonFileIndex() + + jsonIndexCfg, _ := setting.Cfg.GetSection("dashboards.json") + jsonIndexEnabled := jsonIndexCfg.Key("enabled").MustBool(false) + + if jsonIndexEnabled { + jsonFilesPath := jsonIndexCfg.Key("path").String() + if !filepath.IsAbs(jsonFilesPath) { + jsonFilesPath = filepath.Join(setting.HomePath, jsonFilesPath) + } + + orgIds := jsonIndexCfg.Key("org_ids").String() + jsonDashIndex = NewJsonDashIndex(jsonFilesPath, orgIds) + } } func searchHandler(query *Query) error { + hits := make([]*m.DashboardSearchHit, 0) + dashQuery := m.SearchDashboardsQuery{ Title: query.Title, Tag: query.Tag, @@ -35,11 +54,22 @@ func searchHandler(query *Query) error { return err } - if err := setIsStarredFlagOnSearchResults(query.UserId, query.Result); err != nil { + hits = append(hits, dashQuery.Result...) + + if jsonDashIndex != nil { + jsonHits, err := jsonDashIndex.Search(query) + if err != nil { + return err + } + + hits = append(hits, jsonHits...) + } + + if err := setIsStarredFlagOnSearchResults(query.UserId, hits); err != nil { return err } - query.Result = dashQuery.Result + query.Result = hits return nil } diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 8b6f83ec08a..389b6b933b9 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -258,11 +258,13 @@ func loadSpecifedConfigFile(configFile string) { defaultSec, err := Cfg.GetSection(section.Name()) if err != nil { - log.Fatal(3, "Unknown config section %s defined in %s", section.Name(), configFile) + log.Error(3, "Unknown config section %s defined in %s", section.Name(), configFile) + continue } defaultKey, err := defaultSec.GetKey(key.Name()) if err != nil { - log.Fatal(3, "Unknown config key %s defined in section %s, in file", key.Name(), section.Name(), configFile) + log.Error(3, "Unknown config key %s defined in section %s, in file", key.Name(), section.Name(), configFile) + continue } defaultKey.SetValue(key.Value()) }