From fe88658bddf988fb7c9eef4e3dd8cb02b1e725da Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Thu, 7 Dec 2017 03:41:45 +0000 Subject: [PATCH 01/44] Resolves grafana/grafana:#9309 --- pkg/tsdb/cloudwatch/credentials.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/tsdb/cloudwatch/credentials.go b/pkg/tsdb/cloudwatch/credentials.go index 784f3b729ac..9acdf87f5db 100644 --- a/pkg/tsdb/cloudwatch/credentials.go +++ b/pkg/tsdb/cloudwatch/credentials.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" "github.com/aws/aws-sdk-go/aws/credentials/endpointcreds" "github.com/aws/aws-sdk-go/aws/ec2metadata" + "github.com/aws/aws-sdk-go/aws/defaults" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/sts" @@ -128,10 +129,10 @@ func remoteCredProvider(sess *session.Session) credentials.Provider { func ecsCredProvider(sess *session.Session, uri string) credentials.Provider { const host = `169.254.170.2` - c := ec2metadata.New(sess) + d := defaults.Get() return endpointcreds.NewProviderClient( - c.Client.Config, - c.Client.Handlers, + *d.Config, + d.Handlers, fmt.Sprintf("http://%s%s", host, uri), func(p *endpointcreds.Provider) { p.ExpiryWindow = 5 * time.Minute }) } From 50daf7463d09cac933e21e925beaeec7f2a17aae Mon Sep 17 00:00:00 2001 From: Adilet Maratov Date: Thu, 7 Dec 2017 16:13:49 +0600 Subject: [PATCH 02/44] Solves problem with Github authentication restriction by organization membership when the organization's access policy is set to "Access restricted". "Access restricted" policy should not stop user to authenticate. How it is solved: * Take organizations_url field data from user basic data response * Make another request to get all organization the user is a member of (public membership) * Authenticate user if appropriate organization found in that list --- pkg/social/github_oauth.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/social/github_oauth.go b/pkg/social/github_oauth.go index c2a109a43e8..7e348e2363a 100644 --- a/pkg/social/github_oauth.go +++ b/pkg/social/github_oauth.go @@ -58,12 +58,12 @@ func (s *SocialGithub) IsTeamMember(client *http.Client) bool { return false } -func (s *SocialGithub) IsOrganizationMember(client *http.Client) bool { +func (s *SocialGithub) IsOrganizationMember(client *http.Client, organizationsUrl string) bool { if len(s.allowedOrganizations) == 0 { return true } - organizations, err := s.FetchOrganizations(client) + organizations, err := s.FetchOrganizations(client, organizationsUrl) if err != nil { return false } @@ -167,12 +167,12 @@ func (s *SocialGithub) HasMoreRecords(headers http.Header) (string, bool) { } -func (s *SocialGithub) FetchOrganizations(client *http.Client) ([]string, error) { +func (s *SocialGithub) FetchOrganizations(client *http.Client, organizationsUrl string) ([]string, error) { type Record struct { Login string `json:"login"` } - response, err := HttpGet(client, fmt.Sprintf(s.apiUrl+"/orgs")) + response, err := HttpGet(client, organizationsUrl) if err != nil { return nil, fmt.Errorf("Error getting organizations: %s", err) } @@ -193,10 +193,12 @@ func (s *SocialGithub) FetchOrganizations(client *http.Client) ([]string, error) } func (s *SocialGithub) UserInfo(client *http.Client) (*BasicUserInfo, error) { + var data struct { - Id int `json:"id"` - Login string `json:"login"` - Email string `json:"email"` + Id int `json:"id"` + Login string `json:"login"` + Email string `json:"email"` + OrganizationsUrl string `json:"organizations_url"` } response, err := HttpGet(client, s.apiUrl) @@ -219,7 +221,7 @@ func (s *SocialGithub) UserInfo(client *http.Client) (*BasicUserInfo, error) { return nil, ErrMissingTeamMembership } - if !s.IsOrganizationMember(client) { + if !s.IsOrganizationMember(client, data.OrganizationsUrl) { return nil, ErrMissingOrganizationMembership } From d69b63cbc0cf6f76f065280b6ed682209b7599de Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 23 Nov 2017 11:29:06 +0100 Subject: [PATCH 03/44] dashboards as cfg: read first cfg version --- conf/dashboards/dashboards.yaml | 6 + conf/defaults.ini | 3 + conf/sample.ini | 3 + pkg/cmd/grafana-server/server.go | 2 +- pkg/models/dashboards.go | 13 +- .../provisioning/dashboard/config_reader.go | 49 ++++ .../provisioning/dashboard/dashboard.go | 51 +++++ .../provisioning/dashboard/dashboard_test.go | 47 ++++ .../provisioning/dashboard/file_reader.go | 212 ++++++++++++++++++ .../dashboard/file_reader_test.go | 88 ++++++++ .../dashboards-from-disk/dev-dashboards.yaml | 11 + .../broken-dashboards/empty-json.json | 0 .../broken-dashboards/invalid.json | 174 ++++++++++++++ .../folder-one/dashboard1.json | 173 ++++++++++++++ .../folder-one/dashboard2.json | 173 ++++++++++++++ pkg/services/provisioning/dashboard/types.go | 34 +++ pkg/services/provisioning/provisioning.go | 37 ++- pkg/services/sqlstore/dashboard.go | 5 + pkg/setting/setting.go | 2 + 19 files changed, 1075 insertions(+), 8 deletions(-) create mode 100644 conf/dashboards/dashboards.yaml create mode 100644 pkg/services/provisioning/dashboard/config_reader.go create mode 100644 pkg/services/provisioning/dashboard/dashboard.go create mode 100644 pkg/services/provisioning/dashboard/dashboard_test.go create mode 100644 pkg/services/provisioning/dashboard/file_reader.go create mode 100644 pkg/services/provisioning/dashboard/file_reader_test.go create mode 100644 pkg/services/provisioning/dashboard/test-configs/dashboards-from-disk/dev-dashboards.yaml create mode 100644 pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/empty-json.json create mode 100644 pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/invalid.json create mode 100644 pkg/services/provisioning/dashboard/test-dashboards/folder-one/dashboard1.json create mode 100644 pkg/services/provisioning/dashboard/test-dashboards/folder-one/dashboard2.json create mode 100644 pkg/services/provisioning/dashboard/types.go diff --git a/conf/dashboards/dashboards.yaml b/conf/dashboards/dashboards.yaml new file mode 100644 index 00000000000..909a621ca18 --- /dev/null +++ b/conf/dashboards/dashboards.yaml @@ -0,0 +1,6 @@ +- name: 'default' + org_id: 1 + folder: '' + type: file + options: + folder: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/conf/defaults.ini b/conf/defaults.ini index a145d57482b..9dd1857f270 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -23,6 +23,9 @@ plugins = data/plugins # Config files containing datasources that will be configured at startup datasources = conf/datasources +# Config files containing folders to read dashboards from and insert into the database. +dashboards = conf/dashboards + #################################### Server ############################## [server] # Protocol (http, https, socket) diff --git a/conf/sample.ini b/conf/sample.ini index 233a97deef8..44e6f0d4ca3 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -23,6 +23,9 @@ # Config files containing datasources that will be configured at startup ;datasources = conf/datasources +# Config files containing folders to read dashboards from and insert into the database. +;dashboards = conf/dashboards + #################################### Server #################################### [server] # Protocol (http, https, socket) diff --git a/pkg/cmd/grafana-server/server.go b/pkg/cmd/grafana-server/server.go index f5c6b0d1cee..820295a9b88 100644 --- a/pkg/cmd/grafana-server/server.go +++ b/pkg/cmd/grafana-server/server.go @@ -68,7 +68,7 @@ func (g *GrafanaServerImpl) Start() { social.NewOAuthService() plugins.Init() - if err := provisioning.StartUp(setting.DatasourcesPath); err != nil { + if err := provisioning.Init(g.context, setting.HomePath, setting.Cfg); err != nil { logger.Error("Failed to provision Grafana from config", "error", err) g.Shutdown(1, "Startup failed") return diff --git a/pkg/models/dashboards.go b/pkg/models/dashboards.go index 0463e9c209b..4d1d1f1f3d5 100644 --- a/pkg/models/dashboards.go +++ b/pkg/models/dashboards.go @@ -11,11 +11,12 @@ import ( // Typed errors var ( - ErrDashboardNotFound = errors.New("Dashboard not found") - ErrDashboardSnapshotNotFound = errors.New("Dashboard snapshot not found") - ErrDashboardWithSameNameExists = errors.New("A dashboard with the same name already exists") - ErrDashboardVersionMismatch = errors.New("The dashboard has been changed by someone else") - ErrDashboardTitleEmpty = errors.New("Dashboard title cannot be empty") + ErrDashboardNotFound = errors.New("Dashboard not found") + ErrDashboardSnapshotNotFound = errors.New("Dashboard snapshot not found") + ErrDashboardWithSameNameExists = errors.New("A dashboard with the same name already exists") + ErrDashboardVersionMismatch = errors.New("The dashboard has been changed by someone else") + ErrDashboardTitleEmpty = errors.New("Dashboard title cannot be empty") + ErrDashboardContainsInvalidAlertData = errors.New("Invalid alert data. Cannot save dashboard") ) type UpdatePluginDashboardError struct { @@ -139,6 +140,8 @@ type SaveDashboardCommand struct { RestoredFrom int `json:"-"` PluginId string `json:"-"` + UpdatedAt time.Time + Result *Dashboard } diff --git a/pkg/services/provisioning/dashboard/config_reader.go b/pkg/services/provisioning/dashboard/config_reader.go new file mode 100644 index 00000000000..d7ee92a592c --- /dev/null +++ b/pkg/services/provisioning/dashboard/config_reader.go @@ -0,0 +1,49 @@ +package dashboard + +import ( + "io/ioutil" + "path/filepath" + "strings" + + yaml "gopkg.in/yaml.v2" +) + +type configReader struct { + path string +} + +func (cr *configReader) readConfig() ([]*DashboardsAsConfig, error) { + files, err := ioutil.ReadDir(cr.path) + if err != nil { + return nil, err + } + + var dashboards []*DashboardsAsConfig + for _, file := range files { + if !strings.HasSuffix(file.Name(), ".yaml") && !strings.HasSuffix(file.Name(), ".yml") { + continue + } + + filename, _ := filepath.Abs(filepath.Join(cr.path, file.Name())) + yamlFile, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + var datasource []*DashboardsAsConfig + err = yaml.Unmarshal(yamlFile, &datasource) + if err != nil { + return nil, err + } + + dashboards = append(dashboards, datasource...) + } + + for i := range dashboards { + if dashboards[i].OrgId == 0 { + dashboards[i].OrgId = 1 + } + } + + return dashboards, nil +} diff --git a/pkg/services/provisioning/dashboard/dashboard.go b/pkg/services/provisioning/dashboard/dashboard.go new file mode 100644 index 00000000000..22ed5add831 --- /dev/null +++ b/pkg/services/provisioning/dashboard/dashboard.go @@ -0,0 +1,51 @@ +package dashboard + +import ( + "context" + "fmt" + + "github.com/grafana/grafana/pkg/log" +) + +type DashboardProvisioner struct { + cfgReader *configReader + log log.Logger + ctx context.Context +} + +func Provision(ctx context.Context, configDirectory string) (*DashboardProvisioner, error) { + d := &DashboardProvisioner{ + cfgReader: &configReader{path: configDirectory}, + log: log.New("provisioning.dashboard"), + ctx: ctx, + } + + return d, d.Init(ctx) +} + +func (provider *DashboardProvisioner) Init(ctx context.Context) error { + cfgs, err := provider.cfgReader.readConfig() + if err != nil { + return err + } + + for _, cfg := range cfgs { + if cfg.Type == "file" { + fileReader, err := NewDashboardFilereader(cfg, provider.log.New("type", cfg.Type, "name", cfg.Name)) + if err != nil { + return err + } + + // err = fileReader.Init() + // if err != nil { + // provider.log.Error("Failed to load dashboards", "error", err) + // } + + go fileReader.Listen(ctx) + } else { + return fmt.Errorf("type %s is not supported", cfg.Type) + } + } + + return nil +} diff --git a/pkg/services/provisioning/dashboard/dashboard_test.go b/pkg/services/provisioning/dashboard/dashboard_test.go new file mode 100644 index 00000000000..6b1bf3fc6d3 --- /dev/null +++ b/pkg/services/provisioning/dashboard/dashboard_test.go @@ -0,0 +1,47 @@ +package dashboard + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +var ( + simpleDashboardConfig string = "./test-configs/dashboards-from-disk" +) + +func TestDashboardsAsConfig(t *testing.T) { + Convey("Dashboards as configuration", t, func() { + + Convey("Can read config file", func() { + + cfgProvifer := configReader{path: simpleDashboardConfig} + cfg, err := cfgProvifer.readConfig() + if err != nil { + t.Fatalf("readConfig return an error %v", err) + } + + So(len(cfg), ShouldEqual, 2) + + ds := cfg[0] + + So(ds.Name, ShouldEqual, "general dashboards") + So(ds.Type, ShouldEqual, "file") + So(ds.OrgId, ShouldEqual, 2) + So(ds.Folder, ShouldEqual, "developers") + + So(len(ds.Options), ShouldEqual, 1) + So(ds.Options["folder"], ShouldEqual, "/var/lib/grafana/dashboards") + + ds2 := cfg[1] + + So(ds2.Name, ShouldEqual, "default") + So(ds2.Type, ShouldEqual, "file") + So(ds2.OrgId, ShouldEqual, 1) + So(ds2.Folder, ShouldEqual, "") + + So(len(ds2.Options), ShouldEqual, 1) + So(ds2.Options["folder"], ShouldEqual, "/var/lib/grafana/dashboards") + }) + }) +} diff --git a/pkg/services/provisioning/dashboard/file_reader.go b/pkg/services/provisioning/dashboard/file_reader.go new file mode 100644 index 00000000000..c9eacb04aeb --- /dev/null +++ b/pkg/services/provisioning/dashboard/file_reader.go @@ -0,0 +1,212 @@ +package dashboard + +import ( + "context" + "fmt" + "github.com/grafana/grafana/pkg/services/alerting" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/grafana/grafana/pkg/bus" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/models" +) + +type fileReader struct { + Cfg *DashboardsAsConfig + Path string + log log.Logger + dashboardCache *dashboardCache +} + +type dashboardCache struct { + mutex *sync.Mutex + dashboards map[string]*DashboardJson +} + +func newDashboardCache() *dashboardCache { + return &dashboardCache{ + dashboards: map[string]*DashboardJson{}, + mutex: &sync.Mutex{}, + } +} + +func (dc *dashboardCache) addCache(json *DashboardJson) { + dc.mutex.Lock() + defer dc.mutex.Unlock() + dc.dashboards[json.Path] = json +} + +func (dc *dashboardCache) getCache(path string) (*DashboardJson, bool) { + dc.mutex.Lock() + defer dc.mutex.Unlock() + v, exist := dc.dashboards[path] + return v, exist +} + +func NewDashboardFilereader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) { + path, ok := cfg.Options["folder"].(string) + if !ok { + return nil, fmt.Errorf("Failed to load dashboards. folder param is not a string") + } + + if _, err := os.Stat(path); os.IsNotExist(err) { + log.Error("Cannot read directory", "error", err) + } + + return &fileReader{ + Cfg: cfg, + Path: path, + log: log, + dashboardCache: newDashboardCache(), + }, nil +} + +func (fr *fileReader) Listen(ctx context.Context) error { + ticker := time.NewTicker(time.Second * 1) + + if err := fr.walkFolder(); err != nil { + fr.log.Error("failed to search for dashboards", "error", err) + } + + for { + select { + case <-ticker.C: + fr.walkFolder() + case <-ctx.Done(): + return nil + } + } +} + +func (fr *fileReader) walkFolder() error { + if _, err := os.Stat(fr.Path); err != nil { + if os.IsNotExist(err) { + return err + } + } + + return filepath.Walk(fr.Path, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + if f.IsDir() { + if strings.HasPrefix(f.Name(), ".") { + return filepath.SkipDir + } + return nil + } + + if !strings.HasSuffix(f.Name(), ".json") { + return nil + } + + cachedDashboard, exist := fr.dashboardCache.getCache(path) + if exist && cachedDashboard.ModTime == f.ModTime() { + return nil + } + + dash, err := fr.readDashboardFromFile(path) + if err != nil { + fr.log.Error("failed to load dashboard from ", "file", path, "error", err) + return nil + } + + cmd := &models.GetDashboardQuery{Slug: dash.Dashboard.Slug} + err = bus.Dispatch(cmd) + + if err == models.ErrDashboardNotFound { + fr.log.Debug("saving new dashboard", "file", path) + return fr.saveDashboard(dash) + } + + if err != nil { + fr.log.Error("failed to query for dashboard", "slug", dash.Dashboard.Slug, "error", err) + return nil + } + + if cmd.Result.Updated.Unix() >= f.ModTime().Unix() { + fr.log.Debug("already using latest version", "dashboard", dash.Dashboard.Slug) + return nil + } + + fr.log.Debug("no dashboard in cache. Loading dashboard from disk into database.", "file", path) + return fr.saveDashboard(dash) + }) +} + +func (fr *fileReader) readDashboardFromFile(path string) (*DashboardJson, error) { + reader, err := os.Open(path) + if err != nil { + return nil, err + } + defer reader.Close() + + data, err := simplejson.NewFromReader(reader) + if err != nil { + return nil, err + } + + stat, _ := os.Stat(path) + dash := &DashboardJson{} + dash.Dashboard = models.NewDashboardFromJson(data) + dash.TitleLower = strings.ToLower(dash.Dashboard.Title) + dash.Path = path + dash.ModTime = stat.ModTime() + dash.OrgId = fr.Cfg.OrgId + dash.Folder = fr.Cfg.Folder + + if dash.Dashboard.Title == "" { + return nil, models.ErrDashboardTitleEmpty + } + + fr.dashboardCache.addCache(dash) + + return dash, nil +} + +func (fr *fileReader) saveDashboard(dashboardJson *DashboardJson) error { + dash := dashboardJson.Dashboard + + if dash.Title == "" { + return models.ErrDashboardTitleEmpty + } + + validateAlertsCmd := alerting.ValidateDashboardAlertsCommand{ + OrgId: dashboardJson.OrgId, + Dashboard: dash, + } + + if err := bus.Dispatch(&validateAlertsCmd); err != nil { + return models.ErrDashboardContainsInvalidAlertData + } + + cmd := models.SaveDashboardCommand{ + Dashboard: dash.Data, + Message: "Dashboard created from file.", + OrgId: dashboardJson.OrgId, + Overwrite: true, + UpdatedAt: dashboardJson.ModTime, + } + + err := bus.Dispatch(&cmd) + if err != nil { + return err + } + + alertCmd := alerting.UpdateDashboardAlertsCommand{ + OrgId: dashboardJson.OrgId, + Dashboard: cmd.Result, + } + + if err := bus.Dispatch(&alertCmd); err != nil { + return err + } + + return nil +} diff --git a/pkg/services/provisioning/dashboard/file_reader_test.go b/pkg/services/provisioning/dashboard/file_reader_test.go new file mode 100644 index 00000000000..095d0feeb3f --- /dev/null +++ b/pkg/services/provisioning/dashboard/file_reader_test.go @@ -0,0 +1,88 @@ +package dashboard + +import ( + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting" + "testing" + + "github.com/grafana/grafana/pkg/log" + . "github.com/smartystreets/goconvey/convey" +) + +var ( + defaultDashboards string = "./test-dashboards/folder-one" + brokenDashboards string = "./test-dashboards/broken-dashboards" +) + +func TestDashboardFileReader(t *testing.T) { + Convey("Reading dashboards from disk", t, func() { + bus.ClearBusHandlers() + + bus.AddHandler("test", mockGetDashboardQuery) + bus.AddHandler("test", mockValidateDashboardAlertsCommand) + bus.AddHandler("test", mockSaveDashboardCommand) + bus.AddHandler("test", mockUpdateDashboardAlertsCommand) + logger := log.New("test.logger") + + Convey("Can read default dashboard", func() { + cfg := &DashboardsAsConfig{ + Name: "Default", + Type: "file", + OrgId: 1, + Folder: "", + Options: map[string]interface{}{ + "folder": defaultDashboards, + }, + } + reader, err := NewDashboardFilereader(cfg, logger) + So(err, ShouldBeNil) + + err = reader.walkFolder() + So(err, ShouldBeNil) + }) + + Convey("Invalid configuration should return error", func() { + cfg := &DashboardsAsConfig{ + Name: "Default", + Type: "file", + OrgId: 1, + Folder: "", + } + + _, err := NewDashboardFilereader(cfg, logger) + So(err, ShouldNotBeNil) + }) + + Convey("Broken dashboards should not cause error", func() { + cfg := &DashboardsAsConfig{ + Name: "Default", + Type: "file", + OrgId: 1, + Folder: "", + Options: map[string]interface{}{ + "folder": brokenDashboards, + }, + } + + _, err := NewDashboardFilereader(cfg, logger) + So(err, ShouldBeNil) + }) + }) +} + +func mockGetDashboardQuery(cmd *models.GetDashboardQuery) error { + return models.ErrDashboardNotFound +} + +func mockValidateDashboardAlertsCommand(cmd *alerting.ValidateDashboardAlertsCommand) error { + return nil +} + +func mockSaveDashboardCommand(cmd *models.SaveDashboardCommand) error { + return nil +} + +func mockUpdateDashboardAlertsCommand(cmd *alerting.UpdateDashboardAlertsCommand) error { + return nil +} diff --git a/pkg/services/provisioning/dashboard/test-configs/dashboards-from-disk/dev-dashboards.yaml b/pkg/services/provisioning/dashboard/test-configs/dashboards-from-disk/dev-dashboards.yaml new file mode 100644 index 00000000000..67cb383e813 --- /dev/null +++ b/pkg/services/provisioning/dashboard/test-configs/dashboards-from-disk/dev-dashboards.yaml @@ -0,0 +1,11 @@ +- name: 'general dashboards' + org_id: 2 + folder: 'developers' + type: file + options: + folder: /var/lib/grafana/dashboards + +- name: 'default' + type: file + options: + folder: /var/lib/grafana/dashboards diff --git a/pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/empty-json.json b/pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/empty-json.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/invalid.json b/pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/invalid.json new file mode 100644 index 00000000000..1aa388a6e78 --- /dev/null +++ b/pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/invalid.json @@ -0,0 +1,174 @@ +[] +{ + "title": "Grafana", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "rows": [ + { + "title": "New row", + "height": "150px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 1, + "span": 12, + "editable": true, + "type": "text", + "mode": "html", + "content": "
\n \n
", + "style": {}, + "title": "Welcome to" + } + ] + }, + { + "title": "Welcome to Grafana", + "height": "210px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 2, + "span": 6, + "type": "text", + "mode": "html", + "content": "
\n\n
\n
\n \n
\n
\n \n
\n
", + "style": {}, + "title": "Documentation Links" + }, + { + "id": 3, + "span": 6, + "type": "text", + "mode": "html", + "content": "
\n\n
\n
\n
    \n
  • Ctrl+S saves the current dashboard
    • \n
  • Ctrl+F Opens the dashboard finder
    • \n
  • Ctrl+H Hide/show row controls
    • \n
  • Click and drag graph title to move panel
    • \n
  • Hit Escape to exit graph when in fullscreen or edit mode
    • \n
  • Click the colored icon in the legend to change series color
    • \n
  • Ctrl or Shift + Click legend name to hide other series
    • \n
\n
\n
\n", + "style": {}, + "title": "Tips & Shortcuts" + } + ] + }, + { + "title": "test", + "height": "250px", + "editable": true, + "collapse": false, + "panels": [ + { + "id": 4, + "span": 12, + "type": "graph", + "x-axis": true, + "y-axis": true, + "scale": 1, + "y_formats": [ + "short", + "short" + ], + "grid": { + "max": null, + "min": null, + "leftMax": null, + "rightMax": null, + "leftMin": null, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "resolution": 100, + "lines": true, + "fill": 1, + "linewidth": 2, + "dashes": false, + "dashLength": 10, + "spaceLength": 10, + "points": false, + "pointradius": 5, + "bars": false, + "stack": true, + "spyable": true, + "options": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "interactive": true, + "legend_counts": true, + "timezone": "browser", + "percentage": false, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": true + }, + "targets": [ + { + "target": "randomWalk('random walk')", + "function": "mean", + "column": "value" + } + ], + "aliasColors": {}, + "aliasYAxis": {}, + "title": "First Graph (click title to edit)", + "datasource": "graphite", + "renderer": "flot", + "annotate": { + "enable": false + } + } + ] + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "enable": true, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "now": true + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [] + }, + "version": 5 + } + \ No newline at end of file diff --git a/pkg/services/provisioning/dashboard/test-dashboards/folder-one/dashboard1.json b/pkg/services/provisioning/dashboard/test-dashboards/folder-one/dashboard1.json new file mode 100644 index 00000000000..5b6765a4ed6 --- /dev/null +++ b/pkg/services/provisioning/dashboard/test-dashboards/folder-one/dashboard1.json @@ -0,0 +1,173 @@ +{ + "title": "Grafana", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "rows": [ + { + "title": "New row", + "height": "150px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 1, + "span": 12, + "editable": true, + "type": "text", + "mode": "html", + "content": "
\n \n
", + "style": {}, + "title": "Welcome to" + } + ] + }, + { + "title": "Welcome to Grafana", + "height": "210px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 2, + "span": 6, + "type": "text", + "mode": "html", + "content": "
\n\n
\n
\n \n
\n
\n \n
\n
", + "style": {}, + "title": "Documentation Links" + }, + { + "id": 3, + "span": 6, + "type": "text", + "mode": "html", + "content": "
\n\n
\n
\n
    \n
  • Ctrl+S saves the current dashboard
    • \n
  • Ctrl+F Opens the dashboard finder
    • \n
  • Ctrl+H Hide/show row controls
    • \n
  • Click and drag graph title to move panel
    • \n
  • Hit Escape to exit graph when in fullscreen or edit mode
    • \n
  • Click the colored icon in the legend to change series color
    • \n
  • Ctrl or Shift + Click legend name to hide other series
    • \n
\n
\n
\n", + "style": {}, + "title": "Tips & Shortcuts" + } + ] + }, + { + "title": "test", + "height": "250px", + "editable": true, + "collapse": false, + "panels": [ + { + "id": 4, + "span": 12, + "type": "graph", + "x-axis": true, + "y-axis": true, + "scale": 1, + "y_formats": [ + "short", + "short" + ], + "grid": { + "max": null, + "min": null, + "leftMax": null, + "rightMax": null, + "leftMin": null, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "resolution": 100, + "lines": true, + "fill": 1, + "linewidth": 2, + "dashes": false, + "dashLength": 10, + "spaceLength": 10, + "points": false, + "pointradius": 5, + "bars": false, + "stack": true, + "spyable": true, + "options": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "interactive": true, + "legend_counts": true, + "timezone": "browser", + "percentage": false, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": true + }, + "targets": [ + { + "target": "randomWalk('random walk')", + "function": "mean", + "column": "value" + } + ], + "aliasColors": {}, + "aliasYAxis": {}, + "title": "First Graph (click title to edit)", + "datasource": "graphite", + "renderer": "flot", + "annotate": { + "enable": false + } + } + ] + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "enable": true, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "now": true + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [] + }, + "version": 5 + } + \ No newline at end of file diff --git a/pkg/services/provisioning/dashboard/test-dashboards/folder-one/dashboard2.json b/pkg/services/provisioning/dashboard/test-dashboards/folder-one/dashboard2.json new file mode 100644 index 00000000000..5b6765a4ed6 --- /dev/null +++ b/pkg/services/provisioning/dashboard/test-dashboards/folder-one/dashboard2.json @@ -0,0 +1,173 @@ +{ + "title": "Grafana", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "rows": [ + { + "title": "New row", + "height": "150px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 1, + "span": 12, + "editable": true, + "type": "text", + "mode": "html", + "content": "
\n \n
", + "style": {}, + "title": "Welcome to" + } + ] + }, + { + "title": "Welcome to Grafana", + "height": "210px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 2, + "span": 6, + "type": "text", + "mode": "html", + "content": "
\n\n
\n
\n \n
\n
\n \n
\n
", + "style": {}, + "title": "Documentation Links" + }, + { + "id": 3, + "span": 6, + "type": "text", + "mode": "html", + "content": "
\n\n
\n
\n
    \n
  • Ctrl+S saves the current dashboard
    • \n
  • Ctrl+F Opens the dashboard finder
    • \n
  • Ctrl+H Hide/show row controls
    • \n
  • Click and drag graph title to move panel
    • \n
  • Hit Escape to exit graph when in fullscreen or edit mode
    • \n
  • Click the colored icon in the legend to change series color
    • \n
  • Ctrl or Shift + Click legend name to hide other series
    • \n
\n
\n
\n", + "style": {}, + "title": "Tips & Shortcuts" + } + ] + }, + { + "title": "test", + "height": "250px", + "editable": true, + "collapse": false, + "panels": [ + { + "id": 4, + "span": 12, + "type": "graph", + "x-axis": true, + "y-axis": true, + "scale": 1, + "y_formats": [ + "short", + "short" + ], + "grid": { + "max": null, + "min": null, + "leftMax": null, + "rightMax": null, + "leftMin": null, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "resolution": 100, + "lines": true, + "fill": 1, + "linewidth": 2, + "dashes": false, + "dashLength": 10, + "spaceLength": 10, + "points": false, + "pointradius": 5, + "bars": false, + "stack": true, + "spyable": true, + "options": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "interactive": true, + "legend_counts": true, + "timezone": "browser", + "percentage": false, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": true + }, + "targets": [ + { + "target": "randomWalk('random walk')", + "function": "mean", + "column": "value" + } + ], + "aliasColors": {}, + "aliasYAxis": {}, + "title": "First Graph (click title to edit)", + "datasource": "graphite", + "renderer": "flot", + "annotate": { + "enable": false + } + } + ] + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "enable": true, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "now": true + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [] + }, + "version": 5 + } + \ No newline at end of file diff --git a/pkg/services/provisioning/dashboard/types.go b/pkg/services/provisioning/dashboard/types.go new file mode 100644 index 00000000000..cef6b217fee --- /dev/null +++ b/pkg/services/provisioning/dashboard/types.go @@ -0,0 +1,34 @@ +package dashboard + +import ( + "sync" + "time" + + "github.com/grafana/grafana/pkg/models" +) + +type DashboardsAsConfig struct { + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + OrgId int64 `json:"org_id" yaml:"org_id"` + Folder string `json:"folder" yaml:"folder"` + Options map[string]interface{} `json:"options" yaml:"options"` +} + +type DashboardJson struct { + TitleLower string + Path string + OrgId int64 + Folder string + ModTime time.Time + Dashboard *models.Dashboard +} + +type DashboardIndex struct { + mutex *sync.Mutex + + PathToDashboard map[string]*DashboardJson +} + +type InsertDashboard func(cmd *models.Dashboard) error +type UpdateDashboard func(cmd *models.SaveDashboardCommand) error diff --git a/pkg/services/provisioning/provisioning.go b/pkg/services/provisioning/provisioning.go index 1bea60f03e4..51f406dc9d5 100644 --- a/pkg/services/provisioning/provisioning.go +++ b/pkg/services/provisioning/provisioning.go @@ -1,14 +1,47 @@ package provisioning import ( + "context" + "path/filepath" + "github.com/grafana/grafana/pkg/log" + "github.com/grafana/grafana/pkg/services/provisioning/dashboard" "github.com/grafana/grafana/pkg/services/provisioning/datasources" + ini "gopkg.in/ini.v1" ) var ( logger log.Logger = log.New("services.provisioning") ) -func StartUp(datasourcePath string) error { - return datasources.Provision(datasourcePath) +type Provisioner struct { + datasourcePath string + dashboardPath string + bgContext context.Context +} + +func Init(backgroundContext context.Context, homePath string, cfg *ini.File) error { + datasourcePath := makeAbsolute(cfg.Section("paths").Key("datasources").String(), homePath) + if err := datasources.Provision(datasourcePath); err != nil { + return err + } + + dashboardPath := makeAbsolute(cfg.Section("paths").Key("dashboards").String(), homePath) + _, err := dashboard.Provision(backgroundContext, dashboardPath) + if err != nil { + return err + } + + return nil +} + +func (p *Provisioner) Listen() error { + return nil +} + +func makeAbsolute(path string, root string) string { + if filepath.IsAbs(path) { + return path + } + return filepath.Join(root, path) } diff --git a/pkg/services/sqlstore/dashboard.go b/pkg/services/sqlstore/dashboard.go index d91b4a08aa6..cdf9d4eb3c7 100644 --- a/pkg/services/sqlstore/dashboard.go +++ b/pkg/services/sqlstore/dashboard.go @@ -81,6 +81,11 @@ func SaveDashboard(cmd *m.SaveDashboardCommand) error { } else { dash.Version += 1 dash.Data.Set("version", dash.Version) + + if !cmd.UpdatedAt.IsZero() { + dash.Updated = cmd.UpdatedAt + } + affectedRows, err = sess.Id(dash.Id).Update(dash) } diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 2caf7366727..f604cdd680b 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -55,6 +55,7 @@ var ( DataPath string PluginsPath string DatasourcesPath string + DashboardsPath string CustomInitPath = "conf/custom.ini" // Log settings. @@ -475,6 +476,7 @@ func NewConfigContext(args *CommandLineArgs) error { InstanceName = Cfg.Section("").Key("instance_name").MustString("unknown_instance_name") PluginsPath = makeAbsolute(Cfg.Section("paths").Key("plugins").String(), HomePath) DatasourcesPath = makeAbsolute(Cfg.Section("paths").Key("datasources").String(), HomePath) + DashboardsPath = makeAbsolute(Cfg.Section("paths").Key("dashboards").String(), HomePath) server := Cfg.Section("server") AppUrl, AppSubUrl = parseAppUrlAndSubUrl(server) From dc0fb8be06d17707b212c9f79df9221f9f5ad751 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 28 Nov 2017 14:01:10 +0100 Subject: [PATCH 04/44] dashboards as cfg: code cleanup --- .../provisioning/dashboard/dashboard.go | 12 ++++------ .../provisioning/dashboard/file_reader.go | 4 ++-- .../datasources.go | 2 +- .../datasources_test.go | 2 +- .../all-properties/all-properties.yaml | 0 .../test-configs/all-properties/not.yaml.txt | 0 .../test-configs/all-properties/second.yaml | 0 .../test-configs/broken-yaml/broken.yaml | 0 .../double-default/default-1.yaml | 0 .../double-default/default-2.yaml | 0 .../one-datasources.yaml | 0 .../insert-two-delete-two/two-datasources.yml | 0 .../two-datasources/two-datasources.yaml | 0 .../zero-datasources/placeholder-for-git | 0 .../{datasources => datasource}/types.go | 2 +- pkg/services/provisioning/provisioning.go | 23 ++++--------------- 16 files changed, 13 insertions(+), 32 deletions(-) rename pkg/services/provisioning/{datasources => datasource}/datasources.go (99%) rename pkg/services/provisioning/{datasources => datasource}/datasources_test.go (99%) rename pkg/services/provisioning/{datasources => datasource}/test-configs/all-properties/all-properties.yaml (100%) rename pkg/services/provisioning/{datasources => datasource}/test-configs/all-properties/not.yaml.txt (100%) rename pkg/services/provisioning/{datasources => datasource}/test-configs/all-properties/second.yaml (100%) rename pkg/services/provisioning/{datasources => datasource}/test-configs/broken-yaml/broken.yaml (100%) rename pkg/services/provisioning/{datasources => datasource}/test-configs/double-default/default-1.yaml (100%) rename pkg/services/provisioning/{datasources => datasource}/test-configs/double-default/default-2.yaml (100%) rename pkg/services/provisioning/{datasources => datasource}/test-configs/insert-two-delete-two/one-datasources.yaml (100%) rename pkg/services/provisioning/{datasources => datasource}/test-configs/insert-two-delete-two/two-datasources.yml (100%) rename pkg/services/provisioning/{datasources => datasource}/test-configs/two-datasources/two-datasources.yaml (100%) rename pkg/services/provisioning/{datasources => datasource}/test-configs/zero-datasources/placeholder-for-git (100%) rename pkg/services/provisioning/{datasources => datasource}/types.go (99%) diff --git a/pkg/services/provisioning/dashboard/dashboard.go b/pkg/services/provisioning/dashboard/dashboard.go index 22ed5add831..c67c645f242 100644 --- a/pkg/services/provisioning/dashboard/dashboard.go +++ b/pkg/services/provisioning/dashboard/dashboard.go @@ -30,19 +30,15 @@ func (provider *DashboardProvisioner) Init(ctx context.Context) error { } for _, cfg := range cfgs { - if cfg.Type == "file" { + switch cfg.Type { + case "file": fileReader, err := NewDashboardFilereader(cfg, provider.log.New("type", cfg.Type, "name", cfg.Name)) if err != nil { return err } - // err = fileReader.Init() - // if err != nil { - // provider.log.Error("Failed to load dashboards", "error", err) - // } - - go fileReader.Listen(ctx) - } else { + go fileReader.ReadAndListen(ctx) + default: return fmt.Errorf("type %s is not supported", cfg.Type) } } diff --git a/pkg/services/provisioning/dashboard/file_reader.go b/pkg/services/provisioning/dashboard/file_reader.go index c9eacb04aeb..b3fa3e5cfd8 100644 --- a/pkg/services/provisioning/dashboard/file_reader.go +++ b/pkg/services/provisioning/dashboard/file_reader.go @@ -67,8 +67,8 @@ func NewDashboardFilereader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade }, nil } -func (fr *fileReader) Listen(ctx context.Context) error { - ticker := time.NewTicker(time.Second * 1) +func (fr *fileReader) ReadAndListen(ctx context.Context) error { + ticker := time.NewTicker(time.Second * 10) if err := fr.walkFolder(); err != nil { fr.log.Error("failed to search for dashboards", "error", err) diff --git a/pkg/services/provisioning/datasources/datasources.go b/pkg/services/provisioning/datasource/datasources.go similarity index 99% rename from pkg/services/provisioning/datasources/datasources.go rename to pkg/services/provisioning/datasource/datasources.go index 325dbbbd757..be854bfe2bc 100644 --- a/pkg/services/provisioning/datasources/datasources.go +++ b/pkg/services/provisioning/datasource/datasources.go @@ -1,4 +1,4 @@ -package datasources +package datasource import ( "errors" diff --git a/pkg/services/provisioning/datasources/datasources_test.go b/pkg/services/provisioning/datasource/datasources_test.go similarity index 99% rename from pkg/services/provisioning/datasources/datasources_test.go rename to pkg/services/provisioning/datasource/datasources_test.go index f3252c28d9d..d746e794d92 100644 --- a/pkg/services/provisioning/datasources/datasources_test.go +++ b/pkg/services/provisioning/datasource/datasources_test.go @@ -1,4 +1,4 @@ -package datasources +package datasource import ( "testing" diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml b/pkg/services/provisioning/datasource/test-configs/all-properties/all-properties.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml rename to pkg/services/provisioning/datasource/test-configs/all-properties/all-properties.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/not.yaml.txt b/pkg/services/provisioning/datasource/test-configs/all-properties/not.yaml.txt similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/not.yaml.txt rename to pkg/services/provisioning/datasource/test-configs/all-properties/not.yaml.txt diff --git a/pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml b/pkg/services/provisioning/datasource/test-configs/all-properties/second.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml rename to pkg/services/provisioning/datasource/test-configs/all-properties/second.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/broken-yaml/broken.yaml b/pkg/services/provisioning/datasource/test-configs/broken-yaml/broken.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/broken-yaml/broken.yaml rename to pkg/services/provisioning/datasource/test-configs/broken-yaml/broken.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/double-default/default-1.yaml b/pkg/services/provisioning/datasource/test-configs/double-default/default-1.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/double-default/default-1.yaml rename to pkg/services/provisioning/datasource/test-configs/double-default/default-1.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/double-default/default-2.yaml b/pkg/services/provisioning/datasource/test-configs/double-default/default-2.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/double-default/default-2.yaml rename to pkg/services/provisioning/datasource/test-configs/double-default/default-2.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/one-datasources.yaml b/pkg/services/provisioning/datasource/test-configs/insert-two-delete-two/one-datasources.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/one-datasources.yaml rename to pkg/services/provisioning/datasource/test-configs/insert-two-delete-two/one-datasources.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/two-datasources.yml b/pkg/services/provisioning/datasource/test-configs/insert-two-delete-two/two-datasources.yml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/two-datasources.yml rename to pkg/services/provisioning/datasource/test-configs/insert-two-delete-two/two-datasources.yml diff --git a/pkg/services/provisioning/datasources/test-configs/two-datasources/two-datasources.yaml b/pkg/services/provisioning/datasource/test-configs/two-datasources/two-datasources.yaml similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/two-datasources/two-datasources.yaml rename to pkg/services/provisioning/datasource/test-configs/two-datasources/two-datasources.yaml diff --git a/pkg/services/provisioning/datasources/test-configs/zero-datasources/placeholder-for-git b/pkg/services/provisioning/datasource/test-configs/zero-datasources/placeholder-for-git similarity index 100% rename from pkg/services/provisioning/datasources/test-configs/zero-datasources/placeholder-for-git rename to pkg/services/provisioning/datasource/test-configs/zero-datasources/placeholder-for-git diff --git a/pkg/services/provisioning/datasources/types.go b/pkg/services/provisioning/datasource/types.go similarity index 99% rename from pkg/services/provisioning/datasources/types.go rename to pkg/services/provisioning/datasource/types.go index ee2175d6a90..6434074d5d4 100644 --- a/pkg/services/provisioning/datasources/types.go +++ b/pkg/services/provisioning/datasource/types.go @@ -1,4 +1,4 @@ -package datasources +package datasource import "github.com/grafana/grafana/pkg/models" import "github.com/grafana/grafana/pkg/components/simplejson" diff --git a/pkg/services/provisioning/provisioning.go b/pkg/services/provisioning/provisioning.go index 51f406dc9d5..c72ba6dc4f5 100644 --- a/pkg/services/provisioning/provisioning.go +++ b/pkg/services/provisioning/provisioning.go @@ -4,30 +4,19 @@ import ( "context" "path/filepath" - "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/services/provisioning/dashboard" - "github.com/grafana/grafana/pkg/services/provisioning/datasources" + "github.com/grafana/grafana/pkg/services/provisioning/datasource" ini "gopkg.in/ini.v1" ) -var ( - logger log.Logger = log.New("services.provisioning") -) - -type Provisioner struct { - datasourcePath string - dashboardPath string - bgContext context.Context -} - -func Init(backgroundContext context.Context, homePath string, cfg *ini.File) error { +func Init(ctx context.Context, homePath string, cfg *ini.File) error { datasourcePath := makeAbsolute(cfg.Section("paths").Key("datasources").String(), homePath) - if err := datasources.Provision(datasourcePath); err != nil { + if err := datasource.Provision(datasourcePath); err != nil { return err } dashboardPath := makeAbsolute(cfg.Section("paths").Key("dashboards").String(), homePath) - _, err := dashboard.Provision(backgroundContext, dashboardPath) + _, err := dashboard.Provision(ctx, dashboardPath) if err != nil { return err } @@ -35,10 +24,6 @@ func Init(backgroundContext context.Context, homePath string, cfg *ini.File) err return nil } -func (p *Provisioner) Listen() error { - return nil -} - func makeAbsolute(path string, root string) string { if filepath.IsAbs(path) { return path From 7f3a7ea1281bde95ed7f8d7cc38ac512b680b40d Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 28 Nov 2017 14:20:56 +0100 Subject: [PATCH 05/44] dashboards as cfg: more tests --- .../dashboard/file_reader_test.go | 76 +++++++- .../one-dashboard/dashboard1.json | 173 ++++++++++++++++++ 2 files changed, 240 insertions(+), 9 deletions(-) create mode 100644 pkg/services/provisioning/dashboard/test-dashboards/one-dashboard/dashboard1.json diff --git a/pkg/services/provisioning/dashboard/file_reader_test.go b/pkg/services/provisioning/dashboard/file_reader_test.go index 095d0feeb3f..e335ee08aa2 100644 --- a/pkg/services/provisioning/dashboard/file_reader_test.go +++ b/pkg/services/provisioning/dashboard/file_reader_test.go @@ -4,7 +4,9 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" + "os" "testing" + "time" "github.com/grafana/grafana/pkg/log" . "github.com/smartystreets/goconvey/convey" @@ -13,11 +15,15 @@ import ( var ( defaultDashboards string = "./test-dashboards/folder-one" brokenDashboards string = "./test-dashboards/broken-dashboards" + oneDashboard string = "./test-dashboards/one-dashboard" + + fakeRepo *fakeDashboardRepo ) func TestDashboardFileReader(t *testing.T) { Convey("Reading dashboards from disk", t, func() { bus.ClearBusHandlers() + fakeRepo = &fakeDashboardRepo{} bus.AddHandler("test", mockGetDashboardQuery) bus.AddHandler("test", mockValidateDashboardAlertsCommand) @@ -25,21 +31,60 @@ func TestDashboardFileReader(t *testing.T) { bus.AddHandler("test", mockUpdateDashboardAlertsCommand) logger := log.New("test.logger") + cfg := &DashboardsAsConfig{ + Name: "Default", + Type: "file", + OrgId: 1, + Folder: "", + Options: map[string]interface{}{}, + } + Convey("Can read default dashboard", func() { - cfg := &DashboardsAsConfig{ - Name: "Default", - Type: "file", - OrgId: 1, - Folder: "", - Options: map[string]interface{}{ - "folder": defaultDashboards, - }, - } + cfg.Options["folder"] = defaultDashboards + reader, err := NewDashboardFilereader(cfg, logger) So(err, ShouldBeNil) err = reader.walkFolder() So(err, ShouldBeNil) + + So(len(fakeRepo.inserted), ShouldEqual, 2) + }) + + Convey("Should not update dashboards when db is newer", func() { + cfg.Options["folder"] = oneDashboard + + fakeRepo.getDashboard = append(fakeRepo.getDashboard, &models.Dashboard{ + Updated: time.Now().Add(time.Hour), + Slug: "grafana", + }) + + reader, err := NewDashboardFilereader(cfg, logger) + So(err, ShouldBeNil) + + err = reader.walkFolder() + So(err, ShouldBeNil) + + So(len(fakeRepo.inserted), ShouldEqual, 0) + }) + + Convey("Can read default dashboard and replace old version in database", func() { + cfg.Options["folder"] = oneDashboard + + stat, _ := os.Stat(oneDashboard + "/dashboard1.json") + + fakeRepo.getDashboard = append(fakeRepo.getDashboard, &models.Dashboard{ + Updated: stat.ModTime().AddDate(0, 0, -1), + Slug: "grafana", + }) + + reader, err := NewDashboardFilereader(cfg, logger) + So(err, ShouldBeNil) + + err = reader.walkFolder() + So(err, ShouldBeNil) + + So(len(fakeRepo.inserted), ShouldEqual, 1) }) Convey("Invalid configuration should return error", func() { @@ -71,7 +116,19 @@ func TestDashboardFileReader(t *testing.T) { }) } +type fakeDashboardRepo struct { + inserted []*models.SaveDashboardCommand + getDashboard []*models.Dashboard +} + func mockGetDashboardQuery(cmd *models.GetDashboardQuery) error { + for _, d := range fakeRepo.getDashboard { + if d.Slug == cmd.Slug { + cmd.Result = d + return nil + } + } + return models.ErrDashboardNotFound } @@ -80,6 +137,7 @@ func mockValidateDashboardAlertsCommand(cmd *alerting.ValidateDashboardAlertsCom } func mockSaveDashboardCommand(cmd *models.SaveDashboardCommand) error { + fakeRepo.inserted = append(fakeRepo.inserted, cmd) return nil } diff --git a/pkg/services/provisioning/dashboard/test-dashboards/one-dashboard/dashboard1.json b/pkg/services/provisioning/dashboard/test-dashboards/one-dashboard/dashboard1.json new file mode 100644 index 00000000000..5b6765a4ed6 --- /dev/null +++ b/pkg/services/provisioning/dashboard/test-dashboards/one-dashboard/dashboard1.json @@ -0,0 +1,173 @@ +{ + "title": "Grafana", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "rows": [ + { + "title": "New row", + "height": "150px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 1, + "span": 12, + "editable": true, + "type": "text", + "mode": "html", + "content": "
\n \n
", + "style": {}, + "title": "Welcome to" + } + ] + }, + { + "title": "Welcome to Grafana", + "height": "210px", + "collapse": false, + "editable": true, + "panels": [ + { + "id": 2, + "span": 6, + "type": "text", + "mode": "html", + "content": "
\n\n
\n
\n \n
\n
\n \n
\n
", + "style": {}, + "title": "Documentation Links" + }, + { + "id": 3, + "span": 6, + "type": "text", + "mode": "html", + "content": "
\n\n
\n
\n
    \n
  • Ctrl+S saves the current dashboard
    • \n
  • Ctrl+F Opens the dashboard finder
    • \n
  • Ctrl+H Hide/show row controls
    • \n
  • Click and drag graph title to move panel
    • \n
  • Hit Escape to exit graph when in fullscreen or edit mode
    • \n
  • Click the colored icon in the legend to change series color
    • \n
  • Ctrl or Shift + Click legend name to hide other series
    • \n
\n
\n
\n", + "style": {}, + "title": "Tips & Shortcuts" + } + ] + }, + { + "title": "test", + "height": "250px", + "editable": true, + "collapse": false, + "panels": [ + { + "id": 4, + "span": 12, + "type": "graph", + "x-axis": true, + "y-axis": true, + "scale": 1, + "y_formats": [ + "short", + "short" + ], + "grid": { + "max": null, + "min": null, + "leftMax": null, + "rightMax": null, + "leftMin": null, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "resolution": 100, + "lines": true, + "fill": 1, + "linewidth": 2, + "dashes": false, + "dashLength": 10, + "spaceLength": 10, + "points": false, + "pointradius": 5, + "bars": false, + "stack": true, + "spyable": true, + "options": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "interactive": true, + "legend_counts": true, + "timezone": "browser", + "percentage": false, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": true + }, + "targets": [ + { + "target": "randomWalk('random walk')", + "function": "mean", + "column": "value" + } + ], + "aliasColors": {}, + "aliasYAxis": {}, + "title": "First Graph (click title to edit)", + "datasource": "graphite", + "renderer": "flot", + "annotate": { + "enable": false + } + } + ] + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "enable": true, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "now": true + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [] + }, + "version": 5 + } + \ No newline at end of file From 93e1d8a19c8c5573a89cee17b3ef5f743ebc02dd Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 28 Nov 2017 16:57:14 +0100 Subject: [PATCH 06/44] dashboards as cfg: make dashboard none editable by default --- .../provisioning/dashboard/dashboard_test.go | 2 + .../provisioning/dashboard/file_reader.go | 47 +++------------ .../dashboards-from-disk/dev-dashboards.yaml | 1 + pkg/services/provisioning/dashboard/types.go | 59 +++++++++++++++---- 4 files changed, 59 insertions(+), 50 deletions(-) diff --git a/pkg/services/provisioning/dashboard/dashboard_test.go b/pkg/services/provisioning/dashboard/dashboard_test.go index 6b1bf3fc6d3..450d0e62ff5 100644 --- a/pkg/services/provisioning/dashboard/dashboard_test.go +++ b/pkg/services/provisioning/dashboard/dashboard_test.go @@ -29,6 +29,7 @@ func TestDashboardsAsConfig(t *testing.T) { So(ds.Type, ShouldEqual, "file") So(ds.OrgId, ShouldEqual, 2) So(ds.Folder, ShouldEqual, "developers") + So(ds.Editable, ShouldBeTrue) So(len(ds.Options), ShouldEqual, 1) So(ds.Options["folder"], ShouldEqual, "/var/lib/grafana/dashboards") @@ -39,6 +40,7 @@ func TestDashboardsAsConfig(t *testing.T) { So(ds2.Type, ShouldEqual, "file") So(ds2.OrgId, ShouldEqual, 1) So(ds2.Folder, ShouldEqual, "") + So(ds2.Editable, ShouldBeFalse) So(len(ds2.Options), ShouldEqual, 1) So(ds2.Options["folder"], ShouldEqual, "/var/lib/grafana/dashboards") diff --git a/pkg/services/provisioning/dashboard/file_reader.go b/pkg/services/provisioning/dashboard/file_reader.go index b3fa3e5cfd8..d27575523ee 100644 --- a/pkg/services/provisioning/dashboard/file_reader.go +++ b/pkg/services/provisioning/dashboard/file_reader.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" "strings" - "sync" "time" "github.com/grafana/grafana/pkg/bus" @@ -24,31 +23,6 @@ type fileReader struct { dashboardCache *dashboardCache } -type dashboardCache struct { - mutex *sync.Mutex - dashboards map[string]*DashboardJson -} - -func newDashboardCache() *dashboardCache { - return &dashboardCache{ - dashboards: map[string]*DashboardJson{}, - mutex: &sync.Mutex{}, - } -} - -func (dc *dashboardCache) addCache(json *DashboardJson) { - dc.mutex.Lock() - defer dc.mutex.Unlock() - dc.dashboards[json.Path] = json -} - -func (dc *dashboardCache) getCache(path string) (*DashboardJson, bool) { - dc.mutex.Lock() - defer dc.mutex.Unlock() - v, exist := dc.dashboards[path] - return v, exist -} - func NewDashboardFilereader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) { path, ok := cfg.Options["folder"].(string) if !ok { @@ -152,20 +126,17 @@ func (fr *fileReader) readDashboardFromFile(path string) (*DashboardJson, error) return nil, err } - stat, _ := os.Stat(path) - dash := &DashboardJson{} - dash.Dashboard = models.NewDashboardFromJson(data) - dash.TitleLower = strings.ToLower(dash.Dashboard.Title) - dash.Path = path - dash.ModTime = stat.ModTime() - dash.OrgId = fr.Cfg.OrgId - dash.Folder = fr.Cfg.Folder - - if dash.Dashboard.Title == "" { - return nil, models.ErrDashboardTitleEmpty + stat, err := os.Stat(path) + if err != nil { + return nil, err } - fr.dashboardCache.addCache(dash) + dash, err := createDashboardJson(data, stat.ModTime(), fr.Cfg) + if err != nil { + return nil, err + } + + fr.dashboardCache.addCache(path, dash) return dash, nil } diff --git a/pkg/services/provisioning/dashboard/test-configs/dashboards-from-disk/dev-dashboards.yaml b/pkg/services/provisioning/dashboard/test-configs/dashboards-from-disk/dev-dashboards.yaml index 67cb383e813..a7c4a812092 100644 --- a/pkg/services/provisioning/dashboard/test-configs/dashboards-from-disk/dev-dashboards.yaml +++ b/pkg/services/provisioning/dashboard/test-configs/dashboards-from-disk/dev-dashboards.yaml @@ -1,6 +1,7 @@ - name: 'general dashboards' org_id: 2 folder: 'developers' + editable: true type: file options: folder: /var/lib/grafana/dashboards diff --git a/pkg/services/provisioning/dashboard/types.go b/pkg/services/provisioning/dashboard/types.go index cef6b217fee..24e31ac9f0a 100644 --- a/pkg/services/provisioning/dashboard/types.go +++ b/pkg/services/provisioning/dashboard/types.go @@ -1,6 +1,8 @@ package dashboard import ( + "github.com/grafana/grafana/pkg/components/simplejson" + "strings" "sync" "time" @@ -8,27 +10,60 @@ import ( ) type DashboardsAsConfig struct { - Name string `json:"name" yaml:"name"` - Type string `json:"type" yaml:"type"` - OrgId int64 `json:"org_id" yaml:"org_id"` - Folder string `json:"folder" yaml:"folder"` - Options map[string]interface{} `json:"options" yaml:"options"` + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + OrgId int64 `json:"org_id" yaml:"org_id"` + Folder string `json:"folder" yaml:"folder"` + Editable bool `json:"editable" yaml:"editable"` + Options map[string]interface{} `json:"options" yaml:"options"` } type DashboardJson struct { TitleLower string - Path string OrgId int64 Folder string ModTime time.Time Dashboard *models.Dashboard } -type DashboardIndex struct { - mutex *sync.Mutex - - PathToDashboard map[string]*DashboardJson +type dashboardCache struct { + mutex *sync.Mutex + dashboards map[string]*DashboardJson } -type InsertDashboard func(cmd *models.Dashboard) error -type UpdateDashboard func(cmd *models.SaveDashboardCommand) error +func newDashboardCache() *dashboardCache { + return &dashboardCache{ + dashboards: map[string]*DashboardJson{}, + mutex: &sync.Mutex{}, + } +} + +func (dc *dashboardCache) addCache(key string, json *DashboardJson) { + dc.mutex.Lock() + defer dc.mutex.Unlock() + dc.dashboards[key] = json +} + +func (dc *dashboardCache) getCache(key string) (*DashboardJson, bool) { + dc.mutex.Lock() + defer dc.mutex.Unlock() + v, exist := dc.dashboards[key] + return v, exist +} + +func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig) (*DashboardJson, error) { + + dash := &DashboardJson{} + dash.Dashboard = models.NewDashboardFromJson(data) + dash.TitleLower = strings.ToLower(dash.Dashboard.Title) + dash.ModTime = lastModified + dash.OrgId = cfg.OrgId + dash.Folder = cfg.Folder + dash.Dashboard.Data.Set("editable", cfg.Editable) + + if dash.Dashboard.Title == "" { + return nil, models.ErrDashboardTitleEmpty + } + + return dash, nil +} From 09cb0f378b92c04ad9011738cfa8243f697b4ac5 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 30 Nov 2017 17:17:49 +0100 Subject: [PATCH 07/44] dashboards as cfg: minor tweaks --- .../provisioning/dashboard/dashboard.go | 5 +- .../provisioning/dashboard/file_reader.go | 23 ++- .../broken-dashboards/invalid.json | 170 +----------------- 3 files changed, 15 insertions(+), 183 deletions(-) diff --git a/pkg/services/provisioning/dashboard/dashboard.go b/pkg/services/provisioning/dashboard/dashboard.go index c67c645f242..820eaf3c0f5 100644 --- a/pkg/services/provisioning/dashboard/dashboard.go +++ b/pkg/services/provisioning/dashboard/dashboard.go @@ -20,10 +20,11 @@ func Provision(ctx context.Context, configDirectory string) (*DashboardProvision ctx: ctx, } - return d, d.Init(ctx) + err := d.Provision(ctx) + return d, err } -func (provider *DashboardProvisioner) Init(ctx context.Context) error { +func (provider *DashboardProvisioner) Provision(ctx context.Context) error { cfgs, err := provider.cfgReader.readConfig() if err != nil { return err diff --git a/pkg/services/provisioning/dashboard/file_reader.go b/pkg/services/provisioning/dashboard/file_reader.go index d27575523ee..d417a73f0da 100644 --- a/pkg/services/provisioning/dashboard/file_reader.go +++ b/pkg/services/provisioning/dashboard/file_reader.go @@ -42,7 +42,7 @@ func NewDashboardFilereader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade } func (fr *fileReader) ReadAndListen(ctx context.Context) error { - ticker := time.NewTicker(time.Second * 10) + ticker := time.NewTicker(time.Second * 5) if err := fr.walkFolder(); err != nil { fr.log.Error("failed to search for dashboards", "error", err) @@ -105,11 +105,10 @@ func (fr *fileReader) walkFolder() error { } if cmd.Result.Updated.Unix() >= f.ModTime().Unix() { - fr.log.Debug("already using latest version", "dashboard", dash.Dashboard.Slug) return nil } - fr.log.Debug("no dashboard in cache. Loading dashboard from disk into database.", "file", path) + fr.log.Debug("no dashboard in cache. loading dashboard from disk into database.", "file", path) return fr.saveDashboard(dash) }) } @@ -141,16 +140,16 @@ func (fr *fileReader) readDashboardFromFile(path string) (*DashboardJson, error) return dash, nil } -func (fr *fileReader) saveDashboard(dashboardJson *DashboardJson) error { - dash := dashboardJson.Dashboard +func (fr *fileReader) saveDashboard(json *DashboardJson) error { + dashboard := json.Dashboard - if dash.Title == "" { + if dashboard.Title == "" { return models.ErrDashboardTitleEmpty } validateAlertsCmd := alerting.ValidateDashboardAlertsCommand{ - OrgId: dashboardJson.OrgId, - Dashboard: dash, + OrgId: json.OrgId, + Dashboard: dashboard, } if err := bus.Dispatch(&validateAlertsCmd); err != nil { @@ -158,11 +157,11 @@ func (fr *fileReader) saveDashboard(dashboardJson *DashboardJson) error { } cmd := models.SaveDashboardCommand{ - Dashboard: dash.Data, + Dashboard: dashboard.Data, Message: "Dashboard created from file.", - OrgId: dashboardJson.OrgId, + OrgId: json.OrgId, Overwrite: true, - UpdatedAt: dashboardJson.ModTime, + UpdatedAt: json.ModTime, } err := bus.Dispatch(&cmd) @@ -171,7 +170,7 @@ func (fr *fileReader) saveDashboard(dashboardJson *DashboardJson) error { } alertCmd := alerting.UpdateDashboardAlertsCommand{ - OrgId: dashboardJson.OrgId, + OrgId: json.OrgId, Dashboard: cmd.Result, } diff --git a/pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/invalid.json b/pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/invalid.json index 1aa388a6e78..0c5e34c2da7 100644 --- a/pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/invalid.json +++ b/pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/invalid.json @@ -1,174 +1,6 @@ [] { "title": "Grafana", - "tags": [], - "style": "dark", - "timezone": "browser", - "editable": true, - "rows": [ - { - "title": "New row", - "height": "150px", - "collapse": false, - "editable": true, - "panels": [ - { - "id": 1, - "span": 12, - "editable": true, - "type": "text", - "mode": "html", - "content": "
\n \n
", - "style": {}, - "title": "Welcome to" - } - ] - }, - { - "title": "Welcome to Grafana", - "height": "210px", - "collapse": false, - "editable": true, - "panels": [ - { - "id": 2, - "span": 6, - "type": "text", - "mode": "html", - "content": "
\n\n
\n
\n \n
\n
\n \n
\n
", - "style": {}, - "title": "Documentation Links" - }, - { - "id": 3, - "span": 6, - "type": "text", - "mode": "html", - "content": "
\n\n
\n
\n
    \n
  • Ctrl+S saves the current dashboard
    • \n
  • Ctrl+F Opens the dashboard finder
    • \n
  • Ctrl+H Hide/show row controls
    • \n
  • Click and drag graph title to move panel
    • \n
  • Hit Escape to exit graph when in fullscreen or edit mode
    • \n
  • Click the colored icon in the legend to change series color
    • \n
  • Ctrl or Shift + Click legend name to hide other series
    • \n
\n
\n
\n", - "style": {}, - "title": "Tips & Shortcuts" - } - ] - }, - { - "title": "test", - "height": "250px", - "editable": true, - "collapse": false, - "panels": [ - { - "id": 4, - "span": 12, - "type": "graph", - "x-axis": true, - "y-axis": true, - "scale": 1, - "y_formats": [ - "short", - "short" - ], - "grid": { - "max": null, - "min": null, - "leftMax": null, - "rightMax": null, - "leftMin": null, - "rightMin": null, - "threshold1": null, - "threshold2": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "resolution": 100, - "lines": true, - "fill": 1, - "linewidth": 2, - "dashes": false, - "dashLength": 10, - "spaceLength": 10, - "points": false, - "pointradius": 5, - "bars": false, - "stack": true, - "spyable": true, - "options": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false - }, - "interactive": true, - "legend_counts": true, - "timezone": "browser", - "percentage": false, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "query_as_alias": true - }, - "targets": [ - { - "target": "randomWalk('random walk')", - "function": "mean", - "column": "value" - } - ], - "aliasColors": {}, - "aliasYAxis": {}, - "title": "First Graph (click title to edit)", - "datasource": "graphite", - "renderer": "flot", - "annotate": { - "enable": false - } - } - ] - } - ], - "nav": [ - { - "type": "timepicker", - "collapse": false, - "enable": true, - "status": "Stable", - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ], - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "now": true - } - ], - "time": { - "from": "now-6h", - "to": "now" - }, - "templating": { - "list": [] - }, - "version": 5 + } \ No newline at end of file From 74e12c260f70f2a3843a4a3bef62b8736f354233 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 30 Nov 2017 17:35:58 +0100 Subject: [PATCH 08/44] dashboards as cfg: move dashboard saving into its own service --- pkg/services/dashboards/dashboards.go | 67 +++++++++++++++++++ .../provisioning/dashboard/file_reader.go | 49 ++------------ pkg/services/provisioning/dashboard/types.go | 21 ++---- 3 files changed, 78 insertions(+), 59 deletions(-) create mode 100644 pkg/services/dashboards/dashboards.go diff --git a/pkg/services/dashboards/dashboards.go b/pkg/services/dashboards/dashboards.go new file mode 100644 index 00000000000..a5d3c3a4766 --- /dev/null +++ b/pkg/services/dashboards/dashboards.go @@ -0,0 +1,67 @@ +package dashboards + +import ( + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/alerting" + "time" +) + +type Repository interface { + SaveDashboard(*SaveDashboardItem) error +} + +type SaveDashboardItem struct { + TitleLower string + OrgId int64 + Folder string + ModTime time.Time + UserId int64 + Message string + Overwrite bool + Dashboard *models.Dashboard +} + +func SaveDashboard(json *SaveDashboardItem) error { + dashboard := json.Dashboard + + if dashboard.Title == "" { + return models.ErrDashboardTitleEmpty + } + + validateAlertsCmd := alerting.ValidateDashboardAlertsCommand{ + OrgId: json.OrgId, + Dashboard: dashboard, + } + + if err := bus.Dispatch(&validateAlertsCmd); err != nil { + return models.ErrDashboardContainsInvalidAlertData + } + + cmd := models.SaveDashboardCommand{ + Dashboard: dashboard.Data, + Message: json.Message, + OrgId: json.OrgId, + Overwrite: json.Overwrite, + } + + if !json.ModTime.IsZero() { + cmd.UpdatedAt = json.ModTime + } + + err := bus.Dispatch(&cmd) + if err != nil { + return err + } + + alertCmd := alerting.UpdateDashboardAlertsCommand{ + OrgId: json.OrgId, + Dashboard: cmd.Result, + } + + if err := bus.Dispatch(&alertCmd); err != nil { + return err + } + + return nil +} diff --git a/pkg/services/provisioning/dashboard/file_reader.go b/pkg/services/provisioning/dashboard/file_reader.go index d417a73f0da..08dbca81373 100644 --- a/pkg/services/provisioning/dashboard/file_reader.go +++ b/pkg/services/provisioning/dashboard/file_reader.go @@ -3,7 +3,7 @@ package dashboard import ( "context" "fmt" - "github.com/grafana/grafana/pkg/services/alerting" + "github.com/grafana/grafana/pkg/services/dashboards" "os" "path/filepath" "strings" @@ -96,7 +96,7 @@ func (fr *fileReader) walkFolder() error { if err == models.ErrDashboardNotFound { fr.log.Debug("saving new dashboard", "file", path) - return fr.saveDashboard(dash) + return dashboards.SaveDashboard(dash) } if err != nil { @@ -109,11 +109,11 @@ func (fr *fileReader) walkFolder() error { } fr.log.Debug("no dashboard in cache. loading dashboard from disk into database.", "file", path) - return fr.saveDashboard(dash) + return dashboards.SaveDashboard(dash) }) } -func (fr *fileReader) readDashboardFromFile(path string) (*DashboardJson, error) { +func (fr *fileReader) readDashboardFromFile(path string) (*dashboards.SaveDashboardItem, error) { reader, err := os.Open(path) if err != nil { return nil, err @@ -139,44 +139,3 @@ func (fr *fileReader) readDashboardFromFile(path string) (*DashboardJson, error) return dash, nil } - -func (fr *fileReader) saveDashboard(json *DashboardJson) error { - dashboard := json.Dashboard - - if dashboard.Title == "" { - return models.ErrDashboardTitleEmpty - } - - validateAlertsCmd := alerting.ValidateDashboardAlertsCommand{ - OrgId: json.OrgId, - Dashboard: dashboard, - } - - if err := bus.Dispatch(&validateAlertsCmd); err != nil { - return models.ErrDashboardContainsInvalidAlertData - } - - cmd := models.SaveDashboardCommand{ - Dashboard: dashboard.Data, - Message: "Dashboard created from file.", - OrgId: json.OrgId, - Overwrite: true, - UpdatedAt: json.ModTime, - } - - err := bus.Dispatch(&cmd) - if err != nil { - return err - } - - alertCmd := alerting.UpdateDashboardAlertsCommand{ - OrgId: json.OrgId, - Dashboard: cmd.Result, - } - - if err := bus.Dispatch(&alertCmd); err != nil { - return err - } - - return nil -} diff --git a/pkg/services/provisioning/dashboard/types.go b/pkg/services/provisioning/dashboard/types.go index 24e31ac9f0a..9480092f262 100644 --- a/pkg/services/provisioning/dashboard/types.go +++ b/pkg/services/provisioning/dashboard/types.go @@ -2,6 +2,7 @@ package dashboard import ( "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/services/dashboards" "strings" "sync" "time" @@ -18,42 +19,34 @@ type DashboardsAsConfig struct { Options map[string]interface{} `json:"options" yaml:"options"` } -type DashboardJson struct { - TitleLower string - OrgId int64 - Folder string - ModTime time.Time - Dashboard *models.Dashboard -} - type dashboardCache struct { mutex *sync.Mutex - dashboards map[string]*DashboardJson + dashboards map[string]*dashboards.SaveDashboardItem } func newDashboardCache() *dashboardCache { return &dashboardCache{ - dashboards: map[string]*DashboardJson{}, + dashboards: map[string]*dashboards.SaveDashboardItem{}, mutex: &sync.Mutex{}, } } -func (dc *dashboardCache) addCache(key string, json *DashboardJson) { +func (dc *dashboardCache) addCache(key string, json *dashboards.SaveDashboardItem) { dc.mutex.Lock() defer dc.mutex.Unlock() dc.dashboards[key] = json } -func (dc *dashboardCache) getCache(key string) (*DashboardJson, bool) { +func (dc *dashboardCache) getCache(key string) (*dashboards.SaveDashboardItem, bool) { dc.mutex.Lock() defer dc.mutex.Unlock() v, exist := dc.dashboards[key] return v, exist } -func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig) (*DashboardJson, error) { +func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig) (*dashboards.SaveDashboardItem, error) { - dash := &DashboardJson{} + dash := &dashboards.SaveDashboardItem{} dash.Dashboard = models.NewDashboardFromJson(data) dash.TitleLower = strings.ToLower(dash.Dashboard.Title) dash.ModTime = lastModified From 9cebb23e01687fe3eaf42c467c37bd24f33646dd Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 30 Nov 2017 17:43:28 +0100 Subject: [PATCH 09/44] dashboards as cfg: revert minor changes --- pkg/api/dashboard.go | 7 +++++++ .../{dashboard => dashboards}/config_reader.go | 2 +- .../provisioning/{dashboard => dashboards}/dashboard.go | 2 +- .../{dashboard => dashboards}/dashboard_test.go | 2 +- .../provisioning/{dashboard => dashboards}/file_reader.go | 2 +- .../{dashboard => dashboards}/file_reader_test.go | 2 +- .../test-configs/dashboards-from-disk/dev-dashboards.yaml | 0 .../test-dashboards/broken-dashboards/empty-json.json | 0 .../test-dashboards/broken-dashboards/invalid.json | 0 .../test-dashboards/folder-one/dashboard1.json | 0 .../test-dashboards/folder-one/dashboard2.json | 0 .../test-dashboards/one-dashboard/dashboard1.json | 0 .../provisioning/{dashboard => dashboards}/types.go | 2 +- .../{datasource => datasources}/datasources.go | 2 +- .../{datasource => datasources}/datasources_test.go | 2 +- .../test-configs/all-properties/all-properties.yaml | 0 .../test-configs/all-properties/not.yaml.txt | 0 .../test-configs/all-properties/second.yaml | 0 .../test-configs/broken-yaml/broken.yaml | 0 .../test-configs/double-default/default-1.yaml | 0 .../test-configs/double-default/default-2.yaml | 0 .../insert-two-delete-two/one-datasources.yaml | 0 .../insert-two-delete-two/two-datasources.yml | 0 .../test-configs/two-datasources/two-datasources.yaml | 0 .../test-configs/zero-datasources/placeholder-for-git | 0 .../provisioning/{datasource => datasources}/types.go | 2 +- pkg/services/provisioning/provisioning.go | 8 ++++---- 27 files changed, 20 insertions(+), 13 deletions(-) rename pkg/services/provisioning/{dashboard => dashboards}/config_reader.go (97%) rename pkg/services/provisioning/{dashboard => dashboards}/dashboard.go (98%) rename pkg/services/provisioning/{dashboard => dashboards}/dashboard_test.go (98%) rename pkg/services/provisioning/{dashboard => dashboards}/file_reader.go (99%) rename pkg/services/provisioning/{dashboard => dashboards}/file_reader_test.go (99%) rename pkg/services/provisioning/{dashboard => dashboards}/test-configs/dashboards-from-disk/dev-dashboards.yaml (100%) rename pkg/services/provisioning/{dashboard => dashboards}/test-dashboards/broken-dashboards/empty-json.json (100%) rename pkg/services/provisioning/{dashboard => dashboards}/test-dashboards/broken-dashboards/invalid.json (100%) rename pkg/services/provisioning/{dashboard => dashboards}/test-dashboards/folder-one/dashboard1.json (100%) rename pkg/services/provisioning/{dashboard => dashboards}/test-dashboards/folder-one/dashboard2.json (100%) rename pkg/services/provisioning/{dashboard => dashboards}/test-dashboards/one-dashboard/dashboard1.json (100%) rename pkg/services/provisioning/{dashboard => dashboards}/types.go (98%) rename pkg/services/provisioning/{datasource => datasources}/datasources.go (99%) rename pkg/services/provisioning/{datasource => datasources}/datasources_test.go (99%) rename pkg/services/provisioning/{datasource => datasources}/test-configs/all-properties/all-properties.yaml (100%) rename pkg/services/provisioning/{datasource => datasources}/test-configs/all-properties/not.yaml.txt (100%) rename pkg/services/provisioning/{datasource => datasources}/test-configs/all-properties/second.yaml (100%) rename pkg/services/provisioning/{datasource => datasources}/test-configs/broken-yaml/broken.yaml (100%) rename pkg/services/provisioning/{datasource => datasources}/test-configs/double-default/default-1.yaml (100%) rename pkg/services/provisioning/{datasource => datasources}/test-configs/double-default/default-2.yaml (100%) rename pkg/services/provisioning/{datasource => datasources}/test-configs/insert-two-delete-two/one-datasources.yaml (100%) rename pkg/services/provisioning/{datasource => datasources}/test-configs/insert-two-delete-two/two-datasources.yml (100%) rename pkg/services/provisioning/{datasource => datasources}/test-configs/two-datasources/two-datasources.yaml (100%) rename pkg/services/provisioning/{datasource => datasources}/test-configs/zero-datasources/placeholder-for-git (100%) rename pkg/services/provisioning/{datasource => datasources}/types.go (99%) diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index df0cbbd745c..900524a8823 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -3,6 +3,7 @@ package api import ( "encoding/json" "fmt" + //"github.com/grafana/grafana/pkg/services/dashboards" "os" "path" "strings" @@ -124,6 +125,12 @@ func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) Response { dash := cmd.GetDashboardModel() + // dashItem := &dashboards.SaveDashboardItem{ + // Dashboard: dash, + // Message: cmd.Message, + // } + // err := dashboards.SaveDashboard(dashItem) + // Check if Title is empty if dash.Title == "" { return ApiError(400, m.ErrDashboardTitleEmpty.Error(), nil) diff --git a/pkg/services/provisioning/dashboard/config_reader.go b/pkg/services/provisioning/dashboards/config_reader.go similarity index 97% rename from pkg/services/provisioning/dashboard/config_reader.go rename to pkg/services/provisioning/dashboards/config_reader.go index d7ee92a592c..a602ca71df3 100644 --- a/pkg/services/provisioning/dashboard/config_reader.go +++ b/pkg/services/provisioning/dashboards/config_reader.go @@ -1,4 +1,4 @@ -package dashboard +package dashboards import ( "io/ioutil" diff --git a/pkg/services/provisioning/dashboard/dashboard.go b/pkg/services/provisioning/dashboards/dashboard.go similarity index 98% rename from pkg/services/provisioning/dashboard/dashboard.go rename to pkg/services/provisioning/dashboards/dashboard.go index 820eaf3c0f5..e2c802a0024 100644 --- a/pkg/services/provisioning/dashboard/dashboard.go +++ b/pkg/services/provisioning/dashboards/dashboard.go @@ -1,4 +1,4 @@ -package dashboard +package dashboards import ( "context" diff --git a/pkg/services/provisioning/dashboard/dashboard_test.go b/pkg/services/provisioning/dashboards/dashboard_test.go similarity index 98% rename from pkg/services/provisioning/dashboard/dashboard_test.go rename to pkg/services/provisioning/dashboards/dashboard_test.go index 450d0e62ff5..e9ab352a122 100644 --- a/pkg/services/provisioning/dashboard/dashboard_test.go +++ b/pkg/services/provisioning/dashboards/dashboard_test.go @@ -1,4 +1,4 @@ -package dashboard +package dashboards import ( "testing" diff --git a/pkg/services/provisioning/dashboard/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go similarity index 99% rename from pkg/services/provisioning/dashboard/file_reader.go rename to pkg/services/provisioning/dashboards/file_reader.go index 08dbca81373..c15fd7cb521 100644 --- a/pkg/services/provisioning/dashboard/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -1,4 +1,4 @@ -package dashboard +package dashboards import ( "context" diff --git a/pkg/services/provisioning/dashboard/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go similarity index 99% rename from pkg/services/provisioning/dashboard/file_reader_test.go rename to pkg/services/provisioning/dashboards/file_reader_test.go index e335ee08aa2..55c26e47dec 100644 --- a/pkg/services/provisioning/dashboard/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -1,4 +1,4 @@ -package dashboard +package dashboards import ( "github.com/grafana/grafana/pkg/bus" diff --git a/pkg/services/provisioning/dashboard/test-configs/dashboards-from-disk/dev-dashboards.yaml b/pkg/services/provisioning/dashboards/test-configs/dashboards-from-disk/dev-dashboards.yaml similarity index 100% rename from pkg/services/provisioning/dashboard/test-configs/dashboards-from-disk/dev-dashboards.yaml rename to pkg/services/provisioning/dashboards/test-configs/dashboards-from-disk/dev-dashboards.yaml diff --git a/pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/empty-json.json b/pkg/services/provisioning/dashboards/test-dashboards/broken-dashboards/empty-json.json similarity index 100% rename from pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/empty-json.json rename to pkg/services/provisioning/dashboards/test-dashboards/broken-dashboards/empty-json.json diff --git a/pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/invalid.json b/pkg/services/provisioning/dashboards/test-dashboards/broken-dashboards/invalid.json similarity index 100% rename from pkg/services/provisioning/dashboard/test-dashboards/broken-dashboards/invalid.json rename to pkg/services/provisioning/dashboards/test-dashboards/broken-dashboards/invalid.json diff --git a/pkg/services/provisioning/dashboard/test-dashboards/folder-one/dashboard1.json b/pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard1.json similarity index 100% rename from pkg/services/provisioning/dashboard/test-dashboards/folder-one/dashboard1.json rename to pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard1.json diff --git a/pkg/services/provisioning/dashboard/test-dashboards/folder-one/dashboard2.json b/pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard2.json similarity index 100% rename from pkg/services/provisioning/dashboard/test-dashboards/folder-one/dashboard2.json rename to pkg/services/provisioning/dashboards/test-dashboards/folder-one/dashboard2.json diff --git a/pkg/services/provisioning/dashboard/test-dashboards/one-dashboard/dashboard1.json b/pkg/services/provisioning/dashboards/test-dashboards/one-dashboard/dashboard1.json similarity index 100% rename from pkg/services/provisioning/dashboard/test-dashboards/one-dashboard/dashboard1.json rename to pkg/services/provisioning/dashboards/test-dashboards/one-dashboard/dashboard1.json diff --git a/pkg/services/provisioning/dashboard/types.go b/pkg/services/provisioning/dashboards/types.go similarity index 98% rename from pkg/services/provisioning/dashboard/types.go rename to pkg/services/provisioning/dashboards/types.go index 9480092f262..5b73b6d10e3 100644 --- a/pkg/services/provisioning/dashboard/types.go +++ b/pkg/services/provisioning/dashboards/types.go @@ -1,4 +1,4 @@ -package dashboard +package dashboards import ( "github.com/grafana/grafana/pkg/components/simplejson" diff --git a/pkg/services/provisioning/datasource/datasources.go b/pkg/services/provisioning/datasources/datasources.go similarity index 99% rename from pkg/services/provisioning/datasource/datasources.go rename to pkg/services/provisioning/datasources/datasources.go index be854bfe2bc..325dbbbd757 100644 --- a/pkg/services/provisioning/datasource/datasources.go +++ b/pkg/services/provisioning/datasources/datasources.go @@ -1,4 +1,4 @@ -package datasource +package datasources import ( "errors" diff --git a/pkg/services/provisioning/datasource/datasources_test.go b/pkg/services/provisioning/datasources/datasources_test.go similarity index 99% rename from pkg/services/provisioning/datasource/datasources_test.go rename to pkg/services/provisioning/datasources/datasources_test.go index d746e794d92..f3252c28d9d 100644 --- a/pkg/services/provisioning/datasource/datasources_test.go +++ b/pkg/services/provisioning/datasources/datasources_test.go @@ -1,4 +1,4 @@ -package datasource +package datasources import ( "testing" diff --git a/pkg/services/provisioning/datasource/test-configs/all-properties/all-properties.yaml b/pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml similarity index 100% rename from pkg/services/provisioning/datasource/test-configs/all-properties/all-properties.yaml rename to pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml diff --git a/pkg/services/provisioning/datasource/test-configs/all-properties/not.yaml.txt b/pkg/services/provisioning/datasources/test-configs/all-properties/not.yaml.txt similarity index 100% rename from pkg/services/provisioning/datasource/test-configs/all-properties/not.yaml.txt rename to pkg/services/provisioning/datasources/test-configs/all-properties/not.yaml.txt diff --git a/pkg/services/provisioning/datasource/test-configs/all-properties/second.yaml b/pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml similarity index 100% rename from pkg/services/provisioning/datasource/test-configs/all-properties/second.yaml rename to pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml diff --git a/pkg/services/provisioning/datasource/test-configs/broken-yaml/broken.yaml b/pkg/services/provisioning/datasources/test-configs/broken-yaml/broken.yaml similarity index 100% rename from pkg/services/provisioning/datasource/test-configs/broken-yaml/broken.yaml rename to pkg/services/provisioning/datasources/test-configs/broken-yaml/broken.yaml diff --git a/pkg/services/provisioning/datasource/test-configs/double-default/default-1.yaml b/pkg/services/provisioning/datasources/test-configs/double-default/default-1.yaml similarity index 100% rename from pkg/services/provisioning/datasource/test-configs/double-default/default-1.yaml rename to pkg/services/provisioning/datasources/test-configs/double-default/default-1.yaml diff --git a/pkg/services/provisioning/datasource/test-configs/double-default/default-2.yaml b/pkg/services/provisioning/datasources/test-configs/double-default/default-2.yaml similarity index 100% rename from pkg/services/provisioning/datasource/test-configs/double-default/default-2.yaml rename to pkg/services/provisioning/datasources/test-configs/double-default/default-2.yaml diff --git a/pkg/services/provisioning/datasource/test-configs/insert-two-delete-two/one-datasources.yaml b/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/one-datasources.yaml similarity index 100% rename from pkg/services/provisioning/datasource/test-configs/insert-two-delete-two/one-datasources.yaml rename to pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/one-datasources.yaml diff --git a/pkg/services/provisioning/datasource/test-configs/insert-two-delete-two/two-datasources.yml b/pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/two-datasources.yml similarity index 100% rename from pkg/services/provisioning/datasource/test-configs/insert-two-delete-two/two-datasources.yml rename to pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/two-datasources.yml diff --git a/pkg/services/provisioning/datasource/test-configs/two-datasources/two-datasources.yaml b/pkg/services/provisioning/datasources/test-configs/two-datasources/two-datasources.yaml similarity index 100% rename from pkg/services/provisioning/datasource/test-configs/two-datasources/two-datasources.yaml rename to pkg/services/provisioning/datasources/test-configs/two-datasources/two-datasources.yaml diff --git a/pkg/services/provisioning/datasource/test-configs/zero-datasources/placeholder-for-git b/pkg/services/provisioning/datasources/test-configs/zero-datasources/placeholder-for-git similarity index 100% rename from pkg/services/provisioning/datasource/test-configs/zero-datasources/placeholder-for-git rename to pkg/services/provisioning/datasources/test-configs/zero-datasources/placeholder-for-git diff --git a/pkg/services/provisioning/datasource/types.go b/pkg/services/provisioning/datasources/types.go similarity index 99% rename from pkg/services/provisioning/datasource/types.go rename to pkg/services/provisioning/datasources/types.go index 6434074d5d4..ee2175d6a90 100644 --- a/pkg/services/provisioning/datasource/types.go +++ b/pkg/services/provisioning/datasources/types.go @@ -1,4 +1,4 @@ -package datasource +package datasources import "github.com/grafana/grafana/pkg/models" import "github.com/grafana/grafana/pkg/components/simplejson" diff --git a/pkg/services/provisioning/provisioning.go b/pkg/services/provisioning/provisioning.go index c72ba6dc4f5..aa3610b596f 100644 --- a/pkg/services/provisioning/provisioning.go +++ b/pkg/services/provisioning/provisioning.go @@ -4,19 +4,19 @@ import ( "context" "path/filepath" - "github.com/grafana/grafana/pkg/services/provisioning/dashboard" - "github.com/grafana/grafana/pkg/services/provisioning/datasource" + "github.com/grafana/grafana/pkg/services/provisioning/dashboards" + "github.com/grafana/grafana/pkg/services/provisioning/datasources" ini "gopkg.in/ini.v1" ) func Init(ctx context.Context, homePath string, cfg *ini.File) error { datasourcePath := makeAbsolute(cfg.Section("paths").Key("datasources").String(), homePath) - if err := datasource.Provision(datasourcePath); err != nil { + if err := datasources.Provision(datasourcePath); err != nil { return err } dashboardPath := makeAbsolute(cfg.Section("paths").Key("dashboards").String(), homePath) - _, err := dashboard.Provision(ctx, dashboardPath) + _, err := dashboards.Provision(ctx, dashboardPath) if err != nil { return err } From 16f072b32023546a5aa17b0d397063e9d562936f Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 1 Dec 2017 13:50:47 +0100 Subject: [PATCH 10/44] dashboards as cfg: move saving logic for dashboards into its own service --- pkg/api/dashboard.go | 41 +++++++------------ pkg/models/dashboards.go | 1 + pkg/services/dashboards/dashboards.go | 12 +++--- .../provisioning/dashboards/file_reader.go | 6 ++- 4 files changed, 26 insertions(+), 34 deletions(-) diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index 900524a8823..67afad4c010 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -3,11 +3,12 @@ package api import ( "encoding/json" "fmt" - //"github.com/grafana/grafana/pkg/services/dashboards" "os" "path" "strings" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/dashdiffs" @@ -17,7 +18,6 @@ import ( "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/search" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" @@ -125,17 +125,6 @@ func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) Response { dash := cmd.GetDashboardModel() - // dashItem := &dashboards.SaveDashboardItem{ - // Dashboard: dash, - // Message: cmd.Message, - // } - // err := dashboards.SaveDashboard(dashItem) - - // Check if Title is empty - if dash.Title == "" { - return ApiError(400, m.ErrDashboardTitleEmpty.Error(), nil) - } - if dash.Id == 0 { limitReached, err := middleware.QuotaReached(c, "dashboard") if err != nil { @@ -146,17 +135,23 @@ func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) Response { } } - validateAlertsCmd := alerting.ValidateDashboardAlertsCommand{ + dashItem := &dashboards.SaveDashboardItem{ + Dashboard: dash, + Message: cmd.Message, OrgId: c.OrgId, UserId: c.UserId, - Dashboard: dash, } - if err := bus.Dispatch(&validateAlertsCmd); err != nil { + dashboard, err := dashboards.SaveDashboard(dashItem) + + if err == m.ErrDashboardTitleEmpty { + return ApiError(400, m.ErrDashboardTitleEmpty.Error(), nil) + } + + if err == m.ErrDashboardContainsInvalidAlertData { return ApiError(500, "Invalid alert data. Cannot save dashboard", err) } - err := bus.Dispatch(&cmd) if err != nil { if err == m.ErrDashboardWithSameNameExists { return Json(412, util.DynMap{"status": "name-exists", "message": err.Error()}) @@ -178,18 +173,12 @@ func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) Response { return ApiError(500, "Failed to save dashboard", err) } - alertCmd := alerting.UpdateDashboardAlertsCommand{ - OrgId: c.OrgId, - UserId: c.UserId, - Dashboard: cmd.Result, - } - - if err := bus.Dispatch(&alertCmd); err != nil { - return ApiError(500, "Failed to save alerts", err) + if err == m.ErrDashboardFailedToUpdateAlertData { + return ApiError(500, "Invalid alert data. Cannot save dashboard", err) } c.TimeRequest(metrics.M_Api_Dashboard_Save) - return Json(200, util.DynMap{"status": "success", "slug": cmd.Result.Slug, "version": cmd.Result.Version}) + return Json(200, util.DynMap{"status": "success", "slug": dashboard.Slug, "version": dashboard.Version}) } func canEditDashboard(role m.RoleType) bool { diff --git a/pkg/models/dashboards.go b/pkg/models/dashboards.go index 4d1d1f1f3d5..aab6d78db3f 100644 --- a/pkg/models/dashboards.go +++ b/pkg/models/dashboards.go @@ -17,6 +17,7 @@ var ( ErrDashboardVersionMismatch = errors.New("The dashboard has been changed by someone else") ErrDashboardTitleEmpty = errors.New("Dashboard title cannot be empty") ErrDashboardContainsInvalidAlertData = errors.New("Invalid alert data. Cannot save dashboard") + ErrDashboardFailedToUpdateAlertData = errors.New("Failed to save alert data") ) type UpdatePluginDashboardError struct { diff --git a/pkg/services/dashboards/dashboards.go b/pkg/services/dashboards/dashboards.go index a5d3c3a4766..c411f6106c2 100644 --- a/pkg/services/dashboards/dashboards.go +++ b/pkg/services/dashboards/dashboards.go @@ -22,11 +22,11 @@ type SaveDashboardItem struct { Dashboard *models.Dashboard } -func SaveDashboard(json *SaveDashboardItem) error { +func SaveDashboard(json *SaveDashboardItem) (*models.Dashboard, error) { dashboard := json.Dashboard if dashboard.Title == "" { - return models.ErrDashboardTitleEmpty + return nil, models.ErrDashboardTitleEmpty } validateAlertsCmd := alerting.ValidateDashboardAlertsCommand{ @@ -35,7 +35,7 @@ func SaveDashboard(json *SaveDashboardItem) error { } if err := bus.Dispatch(&validateAlertsCmd); err != nil { - return models.ErrDashboardContainsInvalidAlertData + return nil, models.ErrDashboardContainsInvalidAlertData } cmd := models.SaveDashboardCommand{ @@ -51,7 +51,7 @@ func SaveDashboard(json *SaveDashboardItem) error { err := bus.Dispatch(&cmd) if err != nil { - return err + return nil, err } alertCmd := alerting.UpdateDashboardAlertsCommand{ @@ -60,8 +60,8 @@ func SaveDashboard(json *SaveDashboardItem) error { } if err := bus.Dispatch(&alertCmd); err != nil { - return err + return nil, models.ErrDashboardFailedToUpdateAlertData } - return nil + return cmd.Result, nil } diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index c15fd7cb521..1b2feb2d591 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -96,7 +96,8 @@ func (fr *fileReader) walkFolder() error { if err == models.ErrDashboardNotFound { fr.log.Debug("saving new dashboard", "file", path) - return dashboards.SaveDashboard(dash) + _, err = dashboards.SaveDashboard(dash) + return err } if err != nil { @@ -109,7 +110,8 @@ func (fr *fileReader) walkFolder() error { } fr.log.Debug("no dashboard in cache. loading dashboard from disk into database.", "file", path) - return dashboards.SaveDashboard(dash) + _, err = dashboards.SaveDashboard(dash) + return err }) } From f5eac2e91dbee0d4fa926fc27c80a5183ff7b0bc Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 1 Dec 2017 14:20:52 +0100 Subject: [PATCH 11/44] dashboards as cfg: expose dashboard service as interface --- pkg/api/dashboard.go | 2 +- pkg/services/dashboards/dashboards.go | 19 ++++++++++++++++--- .../provisioning/dashboards/file_reader.go | 9 ++++++--- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index 67afad4c010..b2e4d2bf9d3 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -142,7 +142,7 @@ func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) Response { UserId: c.UserId, } - dashboard, err := dashboards.SaveDashboard(dashItem) + dashboard, err := dashboards.GetRepository().SaveDashboard(dashItem) if err == m.ErrDashboardTitleEmpty { return ApiError(400, m.ErrDashboardTitleEmpty.Error(), nil) diff --git a/pkg/services/dashboards/dashboards.go b/pkg/services/dashboards/dashboards.go index c411f6106c2..4f05b1f6663 100644 --- a/pkg/services/dashboards/dashboards.go +++ b/pkg/services/dashboards/dashboards.go @@ -1,14 +1,25 @@ package dashboards import ( + "time" + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" - "time" ) type Repository interface { - SaveDashboard(*SaveDashboardItem) error + SaveDashboard(*SaveDashboardItem) (*models.Dashboard, error) +} + +var repositoryInstance Repository + +func GetRepository() Repository { + return repositoryInstance +} + +func SetRepository(rep Repository) { + repositoryInstance = rep } type SaveDashboardItem struct { @@ -22,7 +33,9 @@ type SaveDashboardItem struct { Dashboard *models.Dashboard } -func SaveDashboard(json *SaveDashboardItem) (*models.Dashboard, error) { +type dashboardRepository struct{} + +func (dr *dashboardRepository) SaveDashboard(json *SaveDashboardItem) (*models.Dashboard, error) { dashboard := json.Dashboard if dashboard.Title == "" { diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 1b2feb2d591..1af4f35de12 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -3,12 +3,13 @@ package dashboards import ( "context" "fmt" - "github.com/grafana/grafana/pkg/services/dashboards" "os" "path/filepath" "strings" "time" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" @@ -21,6 +22,7 @@ type fileReader struct { Path string log log.Logger dashboardCache *dashboardCache + dashboardRepo dashboards.Repository } func NewDashboardFilereader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) { @@ -37,6 +39,7 @@ func NewDashboardFilereader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade Cfg: cfg, Path: path, log: log, + dashboardRepo: dashboards.GetRepository(), dashboardCache: newDashboardCache(), }, nil } @@ -96,7 +99,7 @@ func (fr *fileReader) walkFolder() error { if err == models.ErrDashboardNotFound { fr.log.Debug("saving new dashboard", "file", path) - _, err = dashboards.SaveDashboard(dash) + _, err = fr.dashboardRepo.SaveDashboard(dash) return err } @@ -110,7 +113,7 @@ func (fr *fileReader) walkFolder() error { } fr.log.Debug("no dashboard in cache. loading dashboard from disk into database.", "file", path) - _, err = dashboards.SaveDashboard(dash) + _, err = fr.dashboardRepo.SaveDashboard(dash) return err }) } From 288cc355294119c125a1301fa496a30d25086061 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 1 Dec 2017 14:40:37 +0100 Subject: [PATCH 12/44] dashboards as cfg: use gocache for caching --- pkg/services/dashboards/dashboards.go | 8 ++-- .../provisioning/dashboards/file_reader.go | 45 +++++++++++++------ .../dashboards/file_reader_test.go | 31 +++++-------- pkg/services/provisioning/dashboards/types.go | 33 ++------------ 4 files changed, 52 insertions(+), 65 deletions(-) diff --git a/pkg/services/dashboards/dashboards.go b/pkg/services/dashboards/dashboards.go index 4f05b1f6663..11d77442c7c 100644 --- a/pkg/services/dashboards/dashboards.go +++ b/pkg/services/dashboards/dashboards.go @@ -26,7 +26,7 @@ type SaveDashboardItem struct { TitleLower string OrgId int64 Folder string - ModTime time.Time + UpdatedAt time.Time UserId int64 Message string Overwrite bool @@ -56,10 +56,11 @@ func (dr *dashboardRepository) SaveDashboard(json *SaveDashboardItem) (*models.D Message: json.Message, OrgId: json.OrgId, Overwrite: json.Overwrite, + UserId: json.UserId, } - if !json.ModTime.IsZero() { - cmd.UpdatedAt = json.ModTime + if !json.UpdatedAt.IsZero() { + cmd.UpdatedAt = json.UpdatedAt } err := bus.Dispatch(&cmd) @@ -69,6 +70,7 @@ func (dr *dashboardRepository) SaveDashboard(json *SaveDashboardItem) (*models.D alertCmd := alerting.UpdateDashboardAlertsCommand{ OrgId: json.OrgId, + UserId: json.UserId, Dashboard: cmd.Result, } diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 1af4f35de12..708b2ea681f 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -15,14 +15,15 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" + gocache "github.com/patrickmn/go-cache" ) type fileReader struct { - Cfg *DashboardsAsConfig - Path string - log log.Logger - dashboardCache *dashboardCache - dashboardRepo dashboards.Repository + Cfg *DashboardsAsConfig + Path string + log log.Logger + dashboardRepo dashboards.Repository + cache *gocache.Cache } func NewDashboardFilereader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) { @@ -36,14 +37,32 @@ func NewDashboardFilereader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade } return &fileReader{ - Cfg: cfg, - Path: path, - log: log, - dashboardRepo: dashboards.GetRepository(), - dashboardCache: newDashboardCache(), + Cfg: cfg, + Path: path, + log: log, + dashboardRepo: dashboards.GetRepository(), + cache: gocache.New(5*time.Minute, 10*time.Minute), }, nil } +func (fr *fileReader) addCache(key string, json *dashboards.SaveDashboardItem) { + fr.cache.Add(key, json, time.Minute*10) +} + +func (fr *fileReader) getCache(key string) (*dashboards.SaveDashboardItem, bool) { + obj, exist := fr.cache.Get(key) + if !exist { + return nil, exist + } + + dash, ok := obj.(*dashboards.SaveDashboardItem) + if !ok { + return nil, ok + } + + return dash, ok +} + func (fr *fileReader) ReadAndListen(ctx context.Context) error { ticker := time.NewTicker(time.Second * 5) @@ -83,8 +102,8 @@ func (fr *fileReader) walkFolder() error { return nil } - cachedDashboard, exist := fr.dashboardCache.getCache(path) - if exist && cachedDashboard.ModTime == f.ModTime() { + cachedDashboard, exist := fr.getCache(path) + if exist && cachedDashboard.UpdatedAt == f.ModTime() { return nil } @@ -140,7 +159,7 @@ func (fr *fileReader) readDashboardFromFile(path string) (*dashboards.SaveDashbo return nil, err } - fr.dashboardCache.addCache(path, dash) + fr.addCache(path, dash) return dash, nil } diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index 55c26e47dec..ba6e44114fa 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -1,13 +1,14 @@ package dashboards import ( - "github.com/grafana/grafana/pkg/bus" - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/alerting" "os" "testing" "time" + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/log" . "github.com/smartystreets/goconvey/convey" ) @@ -26,9 +27,7 @@ func TestDashboardFileReader(t *testing.T) { fakeRepo = &fakeDashboardRepo{} bus.AddHandler("test", mockGetDashboardQuery) - bus.AddHandler("test", mockValidateDashboardAlertsCommand) - bus.AddHandler("test", mockSaveDashboardCommand) - bus.AddHandler("test", mockUpdateDashboardAlertsCommand) + dashboards.SetRepository(fakeRepo) logger := log.New("test.logger") cfg := &DashboardsAsConfig{ @@ -117,10 +116,15 @@ func TestDashboardFileReader(t *testing.T) { } type fakeDashboardRepo struct { - inserted []*models.SaveDashboardCommand + inserted []*dashboards.SaveDashboardItem getDashboard []*models.Dashboard } +func (repo *fakeDashboardRepo) SaveDashboard(json *dashboards.SaveDashboardItem) (*models.Dashboard, error) { + repo.inserted = append(repo.inserted, json) + return json.Dashboard, nil +} + func mockGetDashboardQuery(cmd *models.GetDashboardQuery) error { for _, d := range fakeRepo.getDashboard { if d.Slug == cmd.Slug { @@ -131,16 +135,3 @@ func mockGetDashboardQuery(cmd *models.GetDashboardQuery) error { return models.ErrDashboardNotFound } - -func mockValidateDashboardAlertsCommand(cmd *alerting.ValidateDashboardAlertsCommand) error { - return nil -} - -func mockSaveDashboardCommand(cmd *models.SaveDashboardCommand) error { - fakeRepo.inserted = append(fakeRepo.inserted, cmd) - return nil -} - -func mockUpdateDashboardAlertsCommand(cmd *alerting.UpdateDashboardAlertsCommand) error { - return nil -} diff --git a/pkg/services/provisioning/dashboards/types.go b/pkg/services/provisioning/dashboards/types.go index 5b73b6d10e3..0a065029a31 100644 --- a/pkg/services/provisioning/dashboards/types.go +++ b/pkg/services/provisioning/dashboards/types.go @@ -1,11 +1,11 @@ package dashboards import ( + "strings" + "time" + "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/services/dashboards" - "strings" - "sync" - "time" "github.com/grafana/grafana/pkg/models" ) @@ -19,37 +19,12 @@ type DashboardsAsConfig struct { Options map[string]interface{} `json:"options" yaml:"options"` } -type dashboardCache struct { - mutex *sync.Mutex - dashboards map[string]*dashboards.SaveDashboardItem -} - -func newDashboardCache() *dashboardCache { - return &dashboardCache{ - dashboards: map[string]*dashboards.SaveDashboardItem{}, - mutex: &sync.Mutex{}, - } -} - -func (dc *dashboardCache) addCache(key string, json *dashboards.SaveDashboardItem) { - dc.mutex.Lock() - defer dc.mutex.Unlock() - dc.dashboards[key] = json -} - -func (dc *dashboardCache) getCache(key string) (*dashboards.SaveDashboardItem, bool) { - dc.mutex.Lock() - defer dc.mutex.Unlock() - v, exist := dc.dashboards[key] - return v, exist -} - func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig) (*dashboards.SaveDashboardItem, error) { dash := &dashboards.SaveDashboardItem{} dash.Dashboard = models.NewDashboardFromJson(data) dash.TitleLower = strings.ToLower(dash.Dashboard.Title) - dash.ModTime = lastModified + dash.UpdatedAt = lastModified dash.OrgId = cfg.OrgId dash.Folder = cfg.Folder dash.Dashboard.Data.Set("editable", cfg.Editable) From 099178466d8325c958c18e736af0e691964e2e68 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 1 Dec 2017 15:10:48 +0100 Subject: [PATCH 13/44] dashboards as cfg: wire up dashboard repo --- pkg/services/dashboards/dashboards.go | 4 ++-- .../provisioning/dashboards/file_reader.go | 18 ++++++++++-------- pkg/services/provisioning/dashboards/types.go | 1 + pkg/services/sqlstore/sqlstore.go | 2 ++ 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/pkg/services/dashboards/dashboards.go b/pkg/services/dashboards/dashboards.go index 11d77442c7c..24e9f240fd8 100644 --- a/pkg/services/dashboards/dashboards.go +++ b/pkg/services/dashboards/dashboards.go @@ -33,9 +33,9 @@ type SaveDashboardItem struct { Dashboard *models.Dashboard } -type dashboardRepository struct{} +type DashboardRepository struct{} -func (dr *dashboardRepository) SaveDashboard(json *SaveDashboardItem) (*models.Dashboard, error) { +func (dr *DashboardRepository) SaveDashboard(json *SaveDashboardItem) (*models.Dashboard, error) { dashboard := json.Dashboard if dashboard.Title == "" { diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 708b2ea681f..bf12eb2e722 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -41,7 +41,7 @@ func NewDashboardFilereader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade Path: path, log: log, dashboardRepo: dashboards.GetRepository(), - cache: gocache.New(5*time.Minute, 10*time.Minute), + cache: gocache.New(5*time.Minute, 30*time.Minute), }, nil } @@ -87,23 +87,23 @@ func (fr *fileReader) walkFolder() error { } } - return filepath.Walk(fr.Path, func(path string, f os.FileInfo, err error) error { + return filepath.Walk(fr.Path, func(path string, fileInfo os.FileInfo, err error) error { if err != nil { return err } - if f.IsDir() { - if strings.HasPrefix(f.Name(), ".") { + if fileInfo.IsDir() { + if strings.HasPrefix(fileInfo.Name(), ".") { return filepath.SkipDir } return nil } - if !strings.HasSuffix(f.Name(), ".json") { + if !strings.HasSuffix(fileInfo.Name(), ".json") { return nil } cachedDashboard, exist := fr.getCache(path) - if exist && cachedDashboard.UpdatedAt == f.ModTime() { + if exist && cachedDashboard.UpdatedAt == fileInfo.ModTime() { return nil } @@ -116,6 +116,7 @@ func (fr *fileReader) walkFolder() error { cmd := &models.GetDashboardQuery{Slug: dash.Dashboard.Slug} err = bus.Dispatch(cmd) + // if we dont have the dashboard in the db, save it! if err == models.ErrDashboardNotFound { fr.log.Debug("saving new dashboard", "file", path) _, err = fr.dashboardRepo.SaveDashboard(dash) @@ -127,11 +128,12 @@ func (fr *fileReader) walkFolder() error { return nil } - if cmd.Result.Updated.Unix() >= f.ModTime().Unix() { + // break if db version is newer then fil version + if cmd.Result.Updated.Unix() >= fileInfo.ModTime().Unix() { return nil } - fr.log.Debug("no dashboard in cache. loading dashboard from disk into database.", "file", path) + fr.log.Debug("loading dashboard from disk into database.", "file", path) _, err = fr.dashboardRepo.SaveDashboard(dash) return err }) diff --git a/pkg/services/provisioning/dashboards/types.go b/pkg/services/provisioning/dashboards/types.go index 0a065029a31..002a56b5f3c 100644 --- a/pkg/services/provisioning/dashboards/types.go +++ b/pkg/services/provisioning/dashboards/types.go @@ -25,6 +25,7 @@ func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *Das dash.Dashboard = models.NewDashboardFromJson(data) dash.TitleLower = strings.ToLower(dash.Dashboard.Title) dash.UpdatedAt = lastModified + dash.Overwrite = true dash.OrgId = cfg.OrgId dash.Folder = cfg.Folder dash.Dashboard.Data.Set("editable", cfg.Editable) diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go index f37499bd60f..2655bf9c22e 100644 --- a/pkg/services/sqlstore/sqlstore.go +++ b/pkg/services/sqlstore/sqlstore.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/grafana/pkg/log" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/annotations" + "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/sqlstore/migrations" "github.com/grafana/grafana/pkg/services/sqlstore/migrator" "github.com/grafana/grafana/pkg/setting" @@ -100,6 +101,7 @@ func SetEngine(engine *xorm.Engine) (err error) { // Init repo instances annotations.SetRepository(&SqlAnnotationRepo{}) + dashboards.SetRepository(&dashboards.DashboardRepository{}) return nil } From 2a18345eb2e01595fb5fa9cd2c0b7f6251f8a8f3 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 1 Dec 2017 15:16:13 +0100 Subject: [PATCH 14/44] dashboards as cfg: disable loading dashboards from disk by default --- conf/dashboards/dashboards.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/conf/dashboards/dashboards.yaml b/conf/dashboards/dashboards.yaml index 909a621ca18..40992d1461e 100644 --- a/conf/dashboards/dashboards.yaml +++ b/conf/dashboards/dashboards.yaml @@ -1,6 +1,6 @@ -- name: 'default' - org_id: 1 - folder: '' - type: file - options: - folder: /var/lib/grafana/dashboards \ No newline at end of file +# - name: 'default' +# org_id: 1 +# folder: '' +# type: file +# options: +# folder: /var/lib/grafana/dashboards \ No newline at end of file From b28ea0b407bfa2d9da9b3f132c968e09b08abebb Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 5 Dec 2017 13:57:39 +0100 Subject: [PATCH 15/44] dashboards as cfg: type --- pkg/services/provisioning/dashboards/dashboard.go | 2 +- pkg/services/provisioning/dashboards/file_reader.go | 2 +- .../provisioning/dashboards/file_reader_test.go | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/services/provisioning/dashboards/dashboard.go b/pkg/services/provisioning/dashboards/dashboard.go index e2c802a0024..1ee0f78497d 100644 --- a/pkg/services/provisioning/dashboards/dashboard.go +++ b/pkg/services/provisioning/dashboards/dashboard.go @@ -33,7 +33,7 @@ func (provider *DashboardProvisioner) Provision(ctx context.Context) error { for _, cfg := range cfgs { switch cfg.Type { case "file": - fileReader, err := NewDashboardFilereader(cfg, provider.log.New("type", cfg.Type, "name", cfg.Name)) + fileReader, err := NewDashboardFileReader(cfg, provider.log.New("type", cfg.Type, "name", cfg.Name)) if err != nil { return err } diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index bf12eb2e722..3491960e329 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -26,7 +26,7 @@ type fileReader struct { cache *gocache.Cache } -func NewDashboardFilereader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) { +func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) { path, ok := cfg.Options["folder"].(string) if !ok { return nil, fmt.Errorf("Failed to load dashboards. folder param is not a string") diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index ba6e44114fa..e91f278322a 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -41,7 +41,7 @@ func TestDashboardFileReader(t *testing.T) { Convey("Can read default dashboard", func() { cfg.Options["folder"] = defaultDashboards - reader, err := NewDashboardFilereader(cfg, logger) + reader, err := NewDashboardFileReader(cfg, logger) So(err, ShouldBeNil) err = reader.walkFolder() @@ -58,7 +58,7 @@ func TestDashboardFileReader(t *testing.T) { Slug: "grafana", }) - reader, err := NewDashboardFilereader(cfg, logger) + reader, err := NewDashboardFileReader(cfg, logger) So(err, ShouldBeNil) err = reader.walkFolder() @@ -77,7 +77,7 @@ func TestDashboardFileReader(t *testing.T) { Slug: "grafana", }) - reader, err := NewDashboardFilereader(cfg, logger) + reader, err := NewDashboardFileReader(cfg, logger) So(err, ShouldBeNil) err = reader.walkFolder() @@ -94,7 +94,7 @@ func TestDashboardFileReader(t *testing.T) { Folder: "", } - _, err := NewDashboardFilereader(cfg, logger) + _, err := NewDashboardFileReader(cfg, logger) So(err, ShouldNotBeNil) }) @@ -109,7 +109,7 @@ func TestDashboardFileReader(t *testing.T) { }, } - _, err := NewDashboardFilereader(cfg, logger) + _, err := NewDashboardFileReader(cfg, logger) So(err, ShouldBeNil) }) }) From 1dfa529941643b486ceadb7f38cfd8f2b1f3d0eb Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 5 Dec 2017 13:58:56 +0100 Subject: [PATCH 16/44] dashboards as cfg: avoid walking fs in parallel --- pkg/services/provisioning/dashboards/file_reader.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index 3491960e329..42f232bca82 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -64,16 +64,24 @@ func (fr *fileReader) getCache(key string) (*dashboards.SaveDashboardItem, bool) } func (fr *fileReader) ReadAndListen(ctx context.Context) error { - ticker := time.NewTicker(time.Second * 5) + ticker := time.NewTicker(time.Second * 3) if err := fr.walkFolder(); err != nil { fr.log.Error("failed to search for dashboards", "error", err) } + running := false + for { select { case <-ticker.C: - fr.walkFolder() + if !running { // avoid walking the filesystem in parallel. incase fs is very slow. + running = true + go func() { + fr.walkFolder() + running = false + }() + } case <-ctx.Done(): return nil } From 361acd3fa1fc3ed5eff413d16f2c74960ec9f9ba Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 5 Dec 2017 15:44:10 +0100 Subject: [PATCH 17/44] dashboards as cfg: include cfg files in dist packages --- conf/dashboards/{dashboards.yaml => sample.yaml} | 0 .../{datasources.yaml => sample.yaml} | 0 packaging/deb/control/postinst | 4 ++++ packaging/deb/default/grafana-server | 4 ++++ packaging/deb/init.d/grafana-server | 4 +++- packaging/deb/systemd/grafana-server.service | 16 ++++++++++------ packaging/mac/bin/grafana | 4 +++- packaging/rpm/control/postinst | 4 ++++ packaging/rpm/init.d/grafana-server | 4 +++- packaging/rpm/sysconfig/grafana-server | 4 ++++ packaging/rpm/systemd/grafana-server.service | 15 +++++++++------ pkg/setting/setting.go | 4 ++-- scripts/grunt/release_task.js | 2 +- 13 files changed, 47 insertions(+), 18 deletions(-) rename conf/dashboards/{dashboards.yaml => sample.yaml} (100%) rename conf/datasources/{datasources.yaml => sample.yaml} (100%) diff --git a/conf/dashboards/dashboards.yaml b/conf/dashboards/sample.yaml similarity index 100% rename from conf/dashboards/dashboards.yaml rename to conf/dashboards/sample.yaml diff --git a/conf/datasources/datasources.yaml b/conf/datasources/sample.yaml similarity index 100% rename from conf/datasources/datasources.yaml rename to conf/datasources/sample.yaml diff --git a/packaging/deb/control/postinst b/packaging/deb/control/postinst index 8e25a0e4124..c71d11bcbde 100755 --- a/packaging/deb/control/postinst +++ b/packaging/deb/control/postinst @@ -29,6 +29,10 @@ case "$1" in if [ ! -f $CONF_FILE ]; then cp /usr/share/grafana/conf/sample.ini $CONF_FILE cp /usr/share/grafana/conf/ldap.toml /etc/grafana/ldap.toml + + mkdir -p /etc/grafana/dashboards /etc/grafana/datasources + cp /usr/share/grafana/conf/dashboards/sample.yaml /etc/grafana/dashboards/sample.yaml + cp /usr/share/grafana/conf/datasources/sample.yaml /etc/grafana/datasources/sample.yaml fi # configuration files should not be modifiable by grafana user, as this can be a security issue diff --git a/packaging/deb/default/grafana-server b/packaging/deb/default/grafana-server index eaa75830d44..b1a59ae1b5f 100644 --- a/packaging/deb/default/grafana-server +++ b/packaging/deb/default/grafana-server @@ -18,5 +18,9 @@ RESTART_ON_UPGRADE=true PLUGINS_DIR=/var/lib/grafana/plugins +DATASOURCES_CFG_DIR=/etc/grafana/datasources + +DASHBOARDS_CFG_DIR=/etc/grafana/dashboards + # Only used on systemd systems PID_FILE_DIR=/var/run/grafana diff --git a/packaging/deb/init.d/grafana-server b/packaging/deb/init.d/grafana-server index 85b0e412d35..44f29c3c6fd 100755 --- a/packaging/deb/init.d/grafana-server +++ b/packaging/deb/init.d/grafana-server @@ -33,6 +33,8 @@ DATA_DIR=/var/lib/grafana PLUGINS_DIR=/var/lib/grafana/plugins LOG_DIR=/var/log/grafana CONF_FILE=$CONF_DIR/grafana.ini +DATASOURCES_CFG_DIR=$CONF_DIR/datasources +DASHBOARDS_CFG_DIR=$CONF_DIR/dashboards MAX_OPEN_FILES=10000 PID_FILE=/var/run/$NAME.pid DAEMON=/usr/sbin/$NAME @@ -55,7 +57,7 @@ if [ -f "$DEFAULT" ]; then . "$DEFAULT" fi -DAEMON_OPTS="--pidfile=${PID_FILE} --config=${CONF_FILE} cfg:default.paths.data=${DATA_DIR} cfg:default.paths.logs=${LOG_DIR} cfg:default.paths.plugins=${PLUGINS_DIR}" +DAEMON_OPTS="--pidfile=${PID_FILE} --config=${CONF_FILE} cfg:default.paths.datasources=$DATASOURCES_CFG_DIR cfg:default.paths.dashboards=$DASHBOARDS_CFG_DIR cfg:default.paths.data=${DATA_DIR} cfg:default.paths.logs=${LOG_DIR} cfg:default.paths.plugins=${PLUGINS_DIR}" function checkUser() { if [ `id -u` -ne 0 ]; then diff --git a/packaging/deb/systemd/grafana-server.service b/packaging/deb/systemd/grafana-server.service index cb7b87932d1..10682a155ee 100644 --- a/packaging/deb/systemd/grafana-server.service +++ b/packaging/deb/systemd/grafana-server.service @@ -14,12 +14,16 @@ Restart=on-failure WorkingDirectory=/usr/share/grafana RuntimeDirectory=grafana RuntimeDirectoryMode=0750 -ExecStart=/usr/sbin/grafana-server \ - --config=${CONF_FILE} \ - --pidfile=${PID_FILE_DIR}/grafana-server.pid \ - cfg:default.paths.logs=${LOG_DIR} \ - cfg:default.paths.data=${DATA_DIR} \ - cfg:default.paths.plugins=${PLUGINS_DIR} +ExecStart=/usr/sbin/grafana-server \ + --config=${CONF_FILE} \ + --pidfile=${PID_FILE_DIR}/grafana-server.pid \ + cfg:default.paths.logs=${LOG_DIR} \ + cfg:default.paths.data=${DATA_DIR} \ + cfg:default.paths.plugins=${PLUGINS_DIR} \ + cfg:default.paths.datasources=${DATASOURCES_CFG_DIR} \ + cfg:default.paths.dashboards=${DASHBOARDS_CFG_DIR} + + LimitNOFILE=10000 TimeoutStopSec=20 UMask=0027 diff --git a/packaging/mac/bin/grafana b/packaging/mac/bin/grafana index fb33079079e..74f4b00662b 100755 --- a/packaging/mac/bin/grafana +++ b/packaging/mac/bin/grafana @@ -6,10 +6,12 @@ HOMEPATH=/usr/local/share/grafana LOGPATH=/usr/local/var/log/grafana DATAPATH=/usr/local/var/lib/grafana PLUGINPATH=/usr/local/var/lib/grafana/plugins +DATASOURCECFGPATH=/usr/local/etc/grafana/datasources +DASHBOARDSCFGPATH=/usr/local/etc/grafana/dashboards case "$1" in start) - $EXECUTABLE --config=$CONFIG --homepath=$HOMEPATH cfg:default.paths.logs=$LOGPATH cfg:default.paths.data=$DATAPATH cfg:default.paths.plugins=$PLUGINPATH 2> /dev/null & + $EXECUTABLE --config=$CONFIG --homepath=$HOMEPATH cfg:default.paths.datasources=$DATASOURCECFGPATH cfg:default.paths.dashboards=$DASHBOARDSCFGPATH cfg:default.paths.logs=$LOGPATH cfg:default.paths.data=$DATAPATH cfg:default.paths.plugins=$PLUGINPATH 2> /dev/null & [ $? -eq 0 ] && echo "$DAEMON started" ;; stop) diff --git a/packaging/rpm/control/postinst b/packaging/rpm/control/postinst index 0bfca949e7f..07ddbc178de 100755 --- a/packaging/rpm/control/postinst +++ b/packaging/rpm/control/postinst @@ -43,6 +43,10 @@ if [ $1 -eq 1 ] ; then if [ ! -f $CONF_FILE ]; then cp /usr/share/grafana/conf/sample.ini $CONF_FILE cp /usr/share/grafana/conf/ldap.toml /etc/grafana/ldap.toml + + mkdir -p /etc/grafana/dashboards /etc/grafana/datasources + cp /usr/share/grafana/conf/dashboards/sample.yaml /etc/grafana/dashboards/sample.yaml + cp /usr/share/grafana/conf/datasources/sample.yaml /etc/grafana/datasources/sample.yaml fi # Set user permissions on /var/log/grafana, /var/lib/grafana diff --git a/packaging/rpm/init.d/grafana-server b/packaging/rpm/init.d/grafana-server index dc63e1ef4c6..eb7d169bb0e 100755 --- a/packaging/rpm/init.d/grafana-server +++ b/packaging/rpm/init.d/grafana-server @@ -32,6 +32,8 @@ DATA_DIR=/var/lib/grafana PLUGINS_DIR=/var/lib/grafana/plugins LOG_DIR=/var/log/grafana CONF_FILE=$CONF_DIR/grafana.ini +DATASOURCES_CFG_DIR=$CONF_DIR/datasources +DASHBOARDS_CFG_DIR=$CONF_DIR/dashboards MAX_OPEN_FILES=10000 PID_FILE=/var/run/$NAME.pid DAEMON=/usr/sbin/$NAME @@ -59,7 +61,7 @@ fi # overwrite settings from default file [ -e /etc/sysconfig/$NAME ] && . /etc/sysconfig/$NAME -DAEMON_OPTS="--pidfile=${PID_FILE} --config=${CONF_FILE} cfg:default.paths.data=${DATA_DIR} cfg:default.paths.logs=${LOG_DIR} cfg:default.paths.plugins=${PLUGINS_DIR}" +DAEMON_OPTS="--pidfile=${PID_FILE} --config=${CONF_FILE} cfg:default.paths.datasources=$DATASOURCES_CFG_DIR cfg:default.paths.dashboards=$DASHBOARDS_CFG_DIR cfg:default.paths.data=${DATA_DIR} cfg:default.paths.logs=${LOG_DIR} cfg:default.paths.plugins=${PLUGINS_DIR}" function isRunning() { status -p $PID_FILE $NAME > /dev/null 2>&1 diff --git a/packaging/rpm/sysconfig/grafana-server b/packaging/rpm/sysconfig/grafana-server index eaa75830d44..b1a59ae1b5f 100644 --- a/packaging/rpm/sysconfig/grafana-server +++ b/packaging/rpm/sysconfig/grafana-server @@ -18,5 +18,9 @@ RESTART_ON_UPGRADE=true PLUGINS_DIR=/var/lib/grafana/plugins +DATASOURCES_CFG_DIR=/etc/grafana/datasources + +DASHBOARDS_CFG_DIR=/etc/grafana/dashboards + # Only used on systemd systems PID_FILE_DIR=/var/run/grafana diff --git a/packaging/rpm/systemd/grafana-server.service b/packaging/rpm/systemd/grafana-server.service index b23e5196e17..e9c93baba6e 100644 --- a/packaging/rpm/systemd/grafana-server.service +++ b/packaging/rpm/systemd/grafana-server.service @@ -14,12 +14,15 @@ Restart=on-failure WorkingDirectory=/usr/share/grafana RuntimeDirectory=grafana RuntimeDirectoryMode=0750 -ExecStart=/usr/sbin/grafana-server \ - --config=${CONF_FILE} \ - --pidfile=${PID_FILE_DIR}/grafana-server.pid \ - cfg:default.paths.logs=${LOG_DIR} \ - cfg:default.paths.data=${DATA_DIR} \ - cfg:default.paths.plugins=${PLUGINS_DIR} +ExecStart=/usr/sbin/grafana-server \ + --config=${CONF_FILE} \ + --pidfile=${PID_FILE_DIR}/grafana-server.pid \ + cfg:default.paths.logs=${LOG_DIR} \ + cfg:default.paths.data=${DATA_DIR} \ + cfg:default.paths.plugins=${PLUGINS_DIR} \ + cfg:default.paths.datasources=${DATASOURCES_CFG_DIR} \ + cfg:default.paths.dashboards=${DASHBOARDS_CFG_DIR} + LimitNOFILE=10000 TimeoutStopSec=20 diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index f604cdd680b..69499c233fe 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -475,8 +475,8 @@ func NewConfigContext(args *CommandLineArgs) error { Env = Cfg.Section("").Key("app_mode").MustString("development") InstanceName = Cfg.Section("").Key("instance_name").MustString("unknown_instance_name") PluginsPath = makeAbsolute(Cfg.Section("paths").Key("plugins").String(), HomePath) - DatasourcesPath = makeAbsolute(Cfg.Section("paths").Key("datasources").String(), HomePath) - DashboardsPath = makeAbsolute(Cfg.Section("paths").Key("dashboards").String(), HomePath) + DatasourcesPath = Cfg.Section("paths").Key("datasources").String() + DashboardsPath = Cfg.Section("paths").Key("dashboards").String() server := Cfg.Section("server") AppUrl, AppSubUrl = parseAppUrlAndSubUrl(server) diff --git a/scripts/grunt/release_task.js b/scripts/grunt/release_task.js index ecce22f20bd..28208ed0086 100644 --- a/scripts/grunt/release_task.js +++ b/scripts/grunt/release_task.js @@ -26,7 +26,7 @@ module.exports = function(grunt) { }); grunt.config('copy.backend_files', { expand: true, - src: ['conf/*', 'vendor/phantomjs/*', 'scripts/*'], + src: ['conf/**', 'vendor/phantomjs/*', 'scripts/*'], options: { mode: true}, dest: '<%= tempDir %>' }); From 50bc801f1ca5f3a85a9818fec1ab33069b6e48b6 Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 6 Dec 2017 10:17:05 +0100 Subject: [PATCH 18/44] dashboards as cfg: copy dash/ds files if missing --- packaging/deb/control/postinst | 10 ++++++---- packaging/rpm/control/postinst | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/packaging/deb/control/postinst b/packaging/deb/control/postinst index c71d11bcbde..b38f9edadef 100755 --- a/packaging/deb/control/postinst +++ b/packaging/deb/control/postinst @@ -29,12 +29,14 @@ case "$1" in if [ ! -f $CONF_FILE ]; then cp /usr/share/grafana/conf/sample.ini $CONF_FILE cp /usr/share/grafana/conf/ldap.toml /etc/grafana/ldap.toml - - mkdir -p /etc/grafana/dashboards /etc/grafana/datasources - cp /usr/share/grafana/conf/dashboards/sample.yaml /etc/grafana/dashboards/sample.yaml - cp /usr/share/grafana/conf/datasources/sample.yaml /etc/grafana/datasources/sample.yaml fi + if [ ! -f $DATASOURCES_CFG_DIR ]; then + mkdir -p /etc/grafana/dashboards /etc/grafana/datasources + cp /usr/share/grafana/conf/dashboards/sample.yaml $DASHBOARDS_CFG_DIR/sample.yaml + cp /usr/share/grafana/conf/datasources/sample.yaml $DATASOURCES_CFG_DIR/sample.yaml + fi + # configuration files should not be modifiable by grafana user, as this can be a security issue chown -Rh root:$GRAFANA_GROUP /etc/grafana/* chmod 755 /etc/grafana diff --git a/packaging/rpm/control/postinst b/packaging/rpm/control/postinst index 07ddbc178de..68e556b7f2d 100755 --- a/packaging/rpm/control/postinst +++ b/packaging/rpm/control/postinst @@ -43,12 +43,14 @@ if [ $1 -eq 1 ] ; then if [ ! -f $CONF_FILE ]; then cp /usr/share/grafana/conf/sample.ini $CONF_FILE cp /usr/share/grafana/conf/ldap.toml /etc/grafana/ldap.toml - - mkdir -p /etc/grafana/dashboards /etc/grafana/datasources - cp /usr/share/grafana/conf/dashboards/sample.yaml /etc/grafana/dashboards/sample.yaml - cp /usr/share/grafana/conf/datasources/sample.yaml /etc/grafana/datasources/sample.yaml fi + if [ ! -f $DATASOURCES_CFG_DIR ]; then + mkdir -p /etc/grafana/dashboards /etc/grafana/datasources + cp /usr/share/grafana/conf/dashboards/sample.yaml $DASHBOARDS_CFG_DIR/sample.yaml + cp /usr/share/grafana/conf/datasources/sample.yaml $DATASOURCES_CFG_DIR/sample.yaml + fi + # Set user permissions on /var/log/grafana, /var/lib/grafana mkdir -p /var/log/grafana /var/lib/grafana chown -R $GRAFANA_USER:$GRAFANA_GROUP /var/log/grafana /var/lib/grafana From 2e610cb25614df7603ece97961b94a717dd9f938 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 7 Dec 2017 15:14:57 +0100 Subject: [PATCH 19/44] dashboards as cfg: move dash/ds config files to /provisioning/* --- conf/defaults.ini | 7 +-- conf/provisioning/dashboards/custom.yaml | 6 +++ .../{ => provisioning}/dashboards/sample.yaml | 0 conf/provisioning/datasources/custom.yaml | 48 +++++++++++++++++++ .../datasources/sample.yaml | 0 packaging/deb/control/postinst | 8 ++-- packaging/deb/default/grafana-server | 4 +- packaging/deb/init.d/grafana-server | 5 +- packaging/deb/systemd/grafana-server.service | 15 +++--- packaging/rpm/control/postinst | 8 ++-- packaging/rpm/init.d/grafana-server | 5 +- packaging/rpm/sysconfig/grafana-server | 4 +- packaging/rpm/systemd/grafana-server.service | 15 +++--- pkg/services/provisioning/provisioning.go | 7 ++- pkg/setting/setting.go | 19 ++++---- 15 files changed, 97 insertions(+), 54 deletions(-) create mode 100644 conf/provisioning/dashboards/custom.yaml rename conf/{ => provisioning}/dashboards/sample.yaml (100%) create mode 100644 conf/provisioning/datasources/custom.yaml rename conf/{ => provisioning}/datasources/sample.yaml (100%) diff --git a/conf/defaults.ini b/conf/defaults.ini index 9dd1857f270..6e099949af1 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -20,11 +20,8 @@ logs = data/log # Directory where grafana will automatically scan and look for plugins plugins = data/plugins -# Config files containing datasources that will be configured at startup -datasources = conf/datasources - -# Config files containing folders to read dashboards from and insert into the database. -dashboards = conf/dashboards +# folder that contains provisioning config files that grafana will apply on startup and while running. +provisioning = conf/provisioning #################################### Server ############################## [server] diff --git a/conf/provisioning/dashboards/custom.yaml b/conf/provisioning/dashboards/custom.yaml new file mode 100644 index 00000000000..909a621ca18 --- /dev/null +++ b/conf/provisioning/dashboards/custom.yaml @@ -0,0 +1,6 @@ +- name: 'default' + org_id: 1 + folder: '' + type: file + options: + folder: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/conf/dashboards/sample.yaml b/conf/provisioning/dashboards/sample.yaml similarity index 100% rename from conf/dashboards/sample.yaml rename to conf/provisioning/dashboards/sample.yaml diff --git a/conf/provisioning/datasources/custom.yaml b/conf/provisioning/datasources/custom.yaml new file mode 100644 index 00000000000..3d3e36ff108 --- /dev/null +++ b/conf/provisioning/datasources/custom.yaml @@ -0,0 +1,48 @@ +# list of datasources that should be deleted from the database +delete_datasources: + - name: Graphite + org_id: 1 + +# list of datasources to insert/update depending +# whats available in the datbase +datasources: + # name of the datasource. Required +- name: Graphite + # datasource type. Required + type: graphite + # access mode. direct or proxy. Required + access: proxy + # org id. will default to org_id 1 if not specified + org_id: 1 + # url + url: http://localhost:8080 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basic_auth: + # basic auth username + basic_auth_user: + # basic auth password + basic_auth_password: + # enable/disable with credentials headers + with_credentials: + # mark as default datasource. Max one per org + is_default: + # fields that will be converted to json and stored in json_data + json_data: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + # json object of data that will be encrypted. + secure_json_data: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: false + diff --git a/conf/datasources/sample.yaml b/conf/provisioning/datasources/sample.yaml similarity index 100% rename from conf/datasources/sample.yaml rename to conf/provisioning/datasources/sample.yaml diff --git a/packaging/deb/control/postinst b/packaging/deb/control/postinst index b38f9edadef..351c966a8e6 100755 --- a/packaging/deb/control/postinst +++ b/packaging/deb/control/postinst @@ -31,10 +31,10 @@ case "$1" in cp /usr/share/grafana/conf/ldap.toml /etc/grafana/ldap.toml fi - if [ ! -f $DATASOURCES_CFG_DIR ]; then - mkdir -p /etc/grafana/dashboards /etc/grafana/datasources - cp /usr/share/grafana/conf/dashboards/sample.yaml $DASHBOARDS_CFG_DIR/sample.yaml - cp /usr/share/grafana/conf/datasources/sample.yaml $DATASOURCES_CFG_DIR/sample.yaml + if [ ! -f $PROVISIONING_CFG_DIR ]; then + mkdir -p $PROVISIONING_CFG_DIR/dashboards $PROVISIONING_CFG_DIR/datasources + cp /usr/share/grafana/conf/provisioning/dashboards/sample.yaml $PROVISIONING_CFG_DIR/dashboards/sample.yaml + cp /usr/share/grafana/conf/provisioning/datasources/sample.yaml $PROVISIONING_CFG_DIR/datasources/sample.yaml fi # configuration files should not be modifiable by grafana user, as this can be a security issue diff --git a/packaging/deb/default/grafana-server b/packaging/deb/default/grafana-server index b1a59ae1b5f..eb77e62d774 100644 --- a/packaging/deb/default/grafana-server +++ b/packaging/deb/default/grafana-server @@ -18,9 +18,7 @@ RESTART_ON_UPGRADE=true PLUGINS_DIR=/var/lib/grafana/plugins -DATASOURCES_CFG_DIR=/etc/grafana/datasources - -DASHBOARDS_CFG_DIR=/etc/grafana/dashboards +PROVISIONING_CFG_DIR=/etc/grafana/provisioning # Only used on systemd systems PID_FILE_DIR=/var/run/grafana diff --git a/packaging/deb/init.d/grafana-server b/packaging/deb/init.d/grafana-server index 44f29c3c6fd..567da94f881 100755 --- a/packaging/deb/init.d/grafana-server +++ b/packaging/deb/init.d/grafana-server @@ -33,8 +33,7 @@ DATA_DIR=/var/lib/grafana PLUGINS_DIR=/var/lib/grafana/plugins LOG_DIR=/var/log/grafana CONF_FILE=$CONF_DIR/grafana.ini -DATASOURCES_CFG_DIR=$CONF_DIR/datasources -DASHBOARDS_CFG_DIR=$CONF_DIR/dashboards +PROVISIONING_CFG_DIR=$CONF_DIR/provisioning MAX_OPEN_FILES=10000 PID_FILE=/var/run/$NAME.pid DAEMON=/usr/sbin/$NAME @@ -57,7 +56,7 @@ if [ -f "$DEFAULT" ]; then . "$DEFAULT" fi -DAEMON_OPTS="--pidfile=${PID_FILE} --config=${CONF_FILE} cfg:default.paths.datasources=$DATASOURCES_CFG_DIR cfg:default.paths.dashboards=$DASHBOARDS_CFG_DIR cfg:default.paths.data=${DATA_DIR} cfg:default.paths.logs=${LOG_DIR} cfg:default.paths.plugins=${PLUGINS_DIR}" +DAEMON_OPTS="--pidfile=${PID_FILE} --config=${CONF_FILE} cfg:default.paths.provisioning=$PROVISIONING_CFG_DIR cfg:default.paths.data=${DATA_DIR} cfg:default.paths.logs=${LOG_DIR} cfg:default.paths.plugins=${PLUGINS_DIR}" function checkUser() { if [ `id -u` -ne 0 ]; then diff --git a/packaging/deb/systemd/grafana-server.service b/packaging/deb/systemd/grafana-server.service index 10682a155ee..acd2a360a93 100644 --- a/packaging/deb/systemd/grafana-server.service +++ b/packaging/deb/systemd/grafana-server.service @@ -14,14 +14,13 @@ Restart=on-failure WorkingDirectory=/usr/share/grafana RuntimeDirectory=grafana RuntimeDirectoryMode=0750 -ExecStart=/usr/sbin/grafana-server \ - --config=${CONF_FILE} \ - --pidfile=${PID_FILE_DIR}/grafana-server.pid \ - cfg:default.paths.logs=${LOG_DIR} \ - cfg:default.paths.data=${DATA_DIR} \ - cfg:default.paths.plugins=${PLUGINS_DIR} \ - cfg:default.paths.datasources=${DATASOURCES_CFG_DIR} \ - cfg:default.paths.dashboards=${DASHBOARDS_CFG_DIR} +ExecStart=/usr/sbin/grafana-server \ + --config=${CONF_FILE} \ + --pidfile=${PID_FILE_DIR}/grafana-server.pid \ + cfg:default.paths.logs=${LOG_DIR} \ + cfg:default.paths.data=${DATA_DIR} \ + cfg:default.paths.plugins=${PLUGINS_DIR} \ + cfg:default.paths.provisioning=${PROVISIONING_CFG_DIR} LimitNOFILE=10000 diff --git a/packaging/rpm/control/postinst b/packaging/rpm/control/postinst index 68e556b7f2d..e75850f258e 100755 --- a/packaging/rpm/control/postinst +++ b/packaging/rpm/control/postinst @@ -45,10 +45,10 @@ if [ $1 -eq 1 ] ; then cp /usr/share/grafana/conf/ldap.toml /etc/grafana/ldap.toml fi - if [ ! -f $DATASOURCES_CFG_DIR ]; then - mkdir -p /etc/grafana/dashboards /etc/grafana/datasources - cp /usr/share/grafana/conf/dashboards/sample.yaml $DASHBOARDS_CFG_DIR/sample.yaml - cp /usr/share/grafana/conf/datasources/sample.yaml $DATASOURCES_CFG_DIR/sample.yaml + if [ ! -f $PROVISIONING_CFG_DIR ]; then + mkdir -p $PROVISIONING_CFG_DIR/dashboards $PROVISIONING_CFG_DIR/datasources + cp /usr/share/grafana/conf/provisioning/dashboards/sample.yaml $PROVISIONING_CFG_DIR/dashboards/sample.yaml + cp /usr/share/grafana/conf/provisioning/datasources/sample.yaml $PROVISIONING_CFG_DIR/datasources/sample.yaml fi # Set user permissions on /var/log/grafana, /var/lib/grafana diff --git a/packaging/rpm/init.d/grafana-server b/packaging/rpm/init.d/grafana-server index eb7d169bb0e..cefe212116c 100755 --- a/packaging/rpm/init.d/grafana-server +++ b/packaging/rpm/init.d/grafana-server @@ -32,8 +32,7 @@ DATA_DIR=/var/lib/grafana PLUGINS_DIR=/var/lib/grafana/plugins LOG_DIR=/var/log/grafana CONF_FILE=$CONF_DIR/grafana.ini -DATASOURCES_CFG_DIR=$CONF_DIR/datasources -DASHBOARDS_CFG_DIR=$CONF_DIR/dashboards +PROVISIONING_CFG_DIR=$CONF_DIR/provisioning MAX_OPEN_FILES=10000 PID_FILE=/var/run/$NAME.pid DAEMON=/usr/sbin/$NAME @@ -61,7 +60,7 @@ fi # overwrite settings from default file [ -e /etc/sysconfig/$NAME ] && . /etc/sysconfig/$NAME -DAEMON_OPTS="--pidfile=${PID_FILE} --config=${CONF_FILE} cfg:default.paths.datasources=$DATASOURCES_CFG_DIR cfg:default.paths.dashboards=$DASHBOARDS_CFG_DIR cfg:default.paths.data=${DATA_DIR} cfg:default.paths.logs=${LOG_DIR} cfg:default.paths.plugins=${PLUGINS_DIR}" +DAEMON_OPTS="--pidfile=${PID_FILE} --config=${CONF_FILE} cfg:default.paths.provisioning=$PROVISIONING_CFG_DIR cfg:default.paths.data=${DATA_DIR} cfg:default.paths.logs=${LOG_DIR} cfg:default.paths.plugins=${PLUGINS_DIR}" function isRunning() { status -p $PID_FILE $NAME > /dev/null 2>&1 diff --git a/packaging/rpm/sysconfig/grafana-server b/packaging/rpm/sysconfig/grafana-server index b1a59ae1b5f..eb77e62d774 100644 --- a/packaging/rpm/sysconfig/grafana-server +++ b/packaging/rpm/sysconfig/grafana-server @@ -18,9 +18,7 @@ RESTART_ON_UPGRADE=true PLUGINS_DIR=/var/lib/grafana/plugins -DATASOURCES_CFG_DIR=/etc/grafana/datasources - -DASHBOARDS_CFG_DIR=/etc/grafana/dashboards +PROVISIONING_CFG_DIR=/etc/grafana/provisioning # Only used on systemd systems PID_FILE_DIR=/var/run/grafana diff --git a/packaging/rpm/systemd/grafana-server.service b/packaging/rpm/systemd/grafana-server.service index e9c93baba6e..f228c8d8b14 100644 --- a/packaging/rpm/systemd/grafana-server.service +++ b/packaging/rpm/systemd/grafana-server.service @@ -14,14 +14,13 @@ Restart=on-failure WorkingDirectory=/usr/share/grafana RuntimeDirectory=grafana RuntimeDirectoryMode=0750 -ExecStart=/usr/sbin/grafana-server \ - --config=${CONF_FILE} \ - --pidfile=${PID_FILE_DIR}/grafana-server.pid \ - cfg:default.paths.logs=${LOG_DIR} \ - cfg:default.paths.data=${DATA_DIR} \ - cfg:default.paths.plugins=${PLUGINS_DIR} \ - cfg:default.paths.datasources=${DATASOURCES_CFG_DIR} \ - cfg:default.paths.dashboards=${DASHBOARDS_CFG_DIR} +ExecStart=/usr/sbin/grafana-server \ + --config=${CONF_FILE} \ + --pidfile=${PID_FILE_DIR}/grafana-server.pid \ + cfg:default.paths.logs=${LOG_DIR} \ + cfg:default.paths.data=${DATA_DIR} \ + cfg:default.paths.plugins=${PLUGINS_DIR} \ + cfg:default.paths.provisioning=${PROVISIONING_CFG_DIR} LimitNOFILE=10000 TimeoutStopSec=20 diff --git a/pkg/services/provisioning/provisioning.go b/pkg/services/provisioning/provisioning.go index aa3610b596f..b41ec37b797 100644 --- a/pkg/services/provisioning/provisioning.go +++ b/pkg/services/provisioning/provisioning.go @@ -2,6 +2,7 @@ package provisioning import ( "context" + "path" "path/filepath" "github.com/grafana/grafana/pkg/services/provisioning/dashboards" @@ -10,12 +11,14 @@ import ( ) func Init(ctx context.Context, homePath string, cfg *ini.File) error { - datasourcePath := makeAbsolute(cfg.Section("paths").Key("datasources").String(), homePath) + provisioningPath := makeAbsolute(cfg.Section("paths").Key("provisioning").String(), homePath) + + datasourcePath := path.Join(provisioningPath, "datasources") if err := datasources.Provision(datasourcePath); err != nil { return err } - dashboardPath := makeAbsolute(cfg.Section("paths").Key("dashboards").String(), homePath) + dashboardPath := path.Join(provisioningPath, "dashboards") _, err := dashboards.Provision(ctx, dashboardPath) if err != nil { return err diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 69499c233fe..ef4a7c42e86 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -50,13 +50,12 @@ var ( BuildStamp int64 // Paths - LogsPath string - HomePath string - DataPath string - PluginsPath string - DatasourcesPath string - DashboardsPath string - CustomInitPath = "conf/custom.ini" + LogsPath string + HomePath string + DataPath string + PluginsPath string + ProvisioningPath string + CustomInitPath = "conf/custom.ini" // Log settings. LogModes []string @@ -475,9 +474,7 @@ func NewConfigContext(args *CommandLineArgs) error { Env = Cfg.Section("").Key("app_mode").MustString("development") InstanceName = Cfg.Section("").Key("instance_name").MustString("unknown_instance_name") PluginsPath = makeAbsolute(Cfg.Section("paths").Key("plugins").String(), HomePath) - DatasourcesPath = Cfg.Section("paths").Key("datasources").String() - DashboardsPath = Cfg.Section("paths").Key("dashboards").String() - + ProvisioningPath = makeAbsolute(Cfg.Section("paths").Key("provisioning").String(), HomePath) server := Cfg.Section("server") AppUrl, AppSubUrl = parseAppUrlAndSubUrl(server) @@ -672,6 +669,6 @@ func LogConfigurationInfo() { logger.Info("Path Data", "path", DataPath) logger.Info("Path Logs", "path", LogsPath) logger.Info("Path Plugins", "path", PluginsPath) - logger.Info("Path Datasources", "path", DatasourcesPath) + logger.Info("Path Provisioning", "path", ProvisioningPath) logger.Info("App mode " + Env) } From 5006f9e4c57ef2c66e5c5a5cb9111f5bfa4c7c3c Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 7 Dec 2017 15:23:14 +0100 Subject: [PATCH 20/44] dashboards as cfg: update docs to use /provisioning --- conf/defaults.ini | 5 ----- conf/sample.ini | 12 ++---------- docs/sources/administration/provisioning.md | 19 ++++++++++++++++++- docs/sources/installation/configuration.md | 8 +++----- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index 6e099949af1..218c91608cc 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -391,11 +391,6 @@ facility = # Syslog tag. By default, the process' argv[0] is used. tag = -#################################### Dashboard JSON files ################ -[dashboards.json] -enabled = false -path = /var/lib/grafana/dashboards - #################################### Usage Quotas ######################## [quota] enabled = false diff --git a/conf/sample.ini b/conf/sample.ini index 44e6f0d4ca3..7107f8354d6 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -20,11 +20,8 @@ # Directory where grafana will automatically scan and look for plugins ;plugins = /var/lib/grafana/plugins -# Config files containing datasources that will be configured at startup -;datasources = conf/datasources - -# Config files containing folders to read dashboards from and insert into the database. -;dashboards = conf/dashboards +# folder that contains provisioning config files that grafana will apply on startup and while running. +; provisioning = conf/provisioning #################################### Server #################################### [server] @@ -370,11 +367,6 @@ log_queries = ;tag = -;#################################### Dashboard JSON files ########################## -[dashboards.json] -;enabled = false -;path = /var/lib/grafana/dashboards - #################################### Alerting ############################ [alerting] # Disable alerting engine & UI features diff --git a/docs/sources/administration/provisioning.md b/docs/sources/administration/provisioning.md index 0745794496f..3dd5c5fd1d3 100644 --- a/docs/sources/administration/provisioning.md +++ b/docs/sources/administration/provisioning.md @@ -74,7 +74,7 @@ Saltstack | [https://github.com/salt-formulas/salt-formula-grafana](https://gith > This feature is available from v5.0 -It's possible to manage datasources in Grafana by adding one or more yaml config files in the [`conf/datasources`](/installation/configuration/#datasources) directory. Each config file can contain a list of `datasources` that will be added or updated during start up. If the datasource already exists, Grafana will update it to match the configuration file. The config file can also contain a list of datasources that should be deleted. That list is called `delete_datasources`. Grafana will delete datasources listed in `delete_datasources` before inserting/updating those in the `datasource` list. +It's possible to manage datasources in Grafana by adding one or more yaml config files in the [`provisioning/datasources`](/installation/configuration/#provisioning) directory. Each config file can contain a list of `datasources` that will be added or updated during start up. If the datasource already exists, Grafana will update it to match the configuration file. The config file can also contain a list of datasources that should be deleted. That list is called `delete_datasources`. Grafana will delete datasources listed in `delete_datasources` before inserting/updating those in the `datasource` list. ### Running multiple grafana instances. If you are running multiple instances of Grafana you might run into problems if they have different versions of the datasource.yaml configuration file. The best way to solve this problem is to add a version number to each datasource in the configuration and increase it when you update the config. Grafana will only update datasources with the same or lower version number than specified in the config. That way old configs cannot overwrite newer configs if they restart at the same time. @@ -165,3 +165,20 @@ Secure json data is a map of settings that will be encrypted with [secret key](/ | tlsClientKey | string | *All* |TLS Client key for outgoing requests | | password | string | Postgre | password | | user | string | Postgre | user | + +### Dashboards + +It's possible to manage dashboards in Grafana by adding one or more yaml config files in the [`provisioning/dashboards`](/installation/configuration/#provisioning) directory. Each config file can contain a list of `dashboards providers` that will load dashboards into grafana. Currently we only support reading dashboards from file but we will add more providers in the future. + +The dashboard provider config file looks like this + +```yaml +- name: 'default' + org_id: 1 + folder: '' + type: file + options: + folder: /var/lib/grafana/dashboards +``` + +When grafana starts it will update/insert all dashboards available in the configured folders. If you modify the file the dashboard will also be updated. \ No newline at end of file diff --git a/docs/sources/installation/configuration.md b/docs/sources/installation/configuration.md index d454455a26b..1333c8c191d 100644 --- a/docs/sources/installation/configuration.md +++ b/docs/sources/installation/configuration.md @@ -91,12 +91,11 @@ file. Directory where grafana will automatically scan and look for plugins -### datasources +### provisioning > This feature is available in 5.0+ -Config files containing datasources that will be configured at startup. -You can read more about the config files at the [provisioning page](/administration/provisioning/#datasources). +Folder that contains [provisioning](/administration/provisioning) config files that grafana will apply on startup. Dashboards will be reloaded when the json files changes ## [server] @@ -635,8 +634,7 @@ Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1. ## [dashboards.json] -If you have a system that automatically builds dashboards as json files you can enable this feature to have the -Grafana backend index those json dashboards which will make them appear in regular dashboard search. +> This have been replaced with dashboards [provisioning](/administration/provisioning) in 5.0+ ### enabled `true` or `false`. Is disabled by default. From 5fb8b372416a053ca3937467ff396e8da1d1a3c7 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 7 Dec 2017 16:12:56 +0100 Subject: [PATCH 21/44] changelog: note about closing #5269 and #9654 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b4adfcdda2..feff341d4d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ # 4.7.0 (unreleased) +## Breaking change if you are running night build +Config files for provisioning datasources as configuration have changed from `/conf/datasources` to `/conf/provisioning/datasources`. +From `/etc/grafana/datasources` to `/etc/grafana/provisioning/datasources` when installed with deb/rpm packages. + ## New Features * **Data Source Proxy**: Add support for whitelisting specified cookies that will be passed through to the data source when proxying data source requests [#5457](https://github.com/grafana/grafana/issues/5457), thanks [@robingustafsson](https://github.com/robingustafsson) * **Postgres/MySQL**: add __timeGroup macro for mysql [#9596](https://github.com/grafana/grafana/pull/9596), thanks [@svenklemm](https://github.com/svenklemm) @@ -17,6 +21,7 @@ * **Datasources**: Its now possible to configure datasources with config files [#1789](https://github.com/grafana/grafana/issues/1789) * **Graphite**: Query editor updated to support new query by tag features [#9230](https://github.com/grafana/grafana/issues/9230) * **Dashboard history**: New config file option versions_to_keep sets how many versions per dashboard to store, [#9671](https://github.com/grafana/grafana/issues/9671) +* **Dashboard as cfg**: Load dashboards from file into Grafana on startup/change [#9654](https://github.com/grafana/grafana/issues/9654) [#5269](https://github.com/grafana/grafana/issues/5269) ## Minor * **Alert panel**: Adds placeholder text when no alerts are within the time range [#9624](https://github.com/grafana/grafana/issues/9624), thx [@straend](https://github.com/straend) From 5aab6b5c205a35dc7ecc1d9185e62a4c8e525247 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 7 Dec 2017 16:26:07 +0100 Subject: [PATCH 22/44] removes last pieces of dashboard.json --- CHANGELOG.md | 3 + pkg/api/api.go | 1 - pkg/api/dashboard.go | 17 --- pkg/services/search/handlers.go | 40 ------- pkg/services/search/handlers_test.go | 1 - pkg/services/search/json_index.go | 140 ------------------------- pkg/services/search/json_index_test.go | 42 -------- 7 files changed, 3 insertions(+), 241 deletions(-) delete mode 100644 pkg/services/search/json_index.go delete mode 100644 pkg/services/search/json_index_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index feff341d4d1..04dccd60276 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ # 4.7.0 (unreleased) ## Breaking change if you are running night build + +[dashboard.json] have been replaced with [dashboard provisioning](http://docs.grafana.org/administration/provisioning/). + Config files for provisioning datasources as configuration have changed from `/conf/datasources` to `/conf/provisioning/datasources`. From `/etc/grafana/datasources` to `/etc/grafana/provisioning/datasources` when installed with deb/rpm packages. diff --git a/pkg/api/api.go b/pkg/api/api.go index 1c57a3e3dba..3810f87dbb8 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -234,7 +234,6 @@ func (hs *HttpServer) registerRoutes() { dashboardRoute.Post("/calculate-diff", bind(dtos.CalculateDiffOptions{}), wrap(CalculateDashboardDiff)) dashboardRoute.Post("/db", reqEditorRole, bind(m.SaveDashboardCommand{}), wrap(PostDashboard)) - dashboardRoute.Get("/file/:file", GetDashboardFromJsonFile) dashboardRoute.Get("/home", wrap(GetHomeDashboard)) dashboardRoute.Get("/tags", GetDashboardTags) dashboardRoute.Post("/import", bind(dtos.ImportDashboardCommand{}), wrap(ImportDashboard)) diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index b2e4d2bf9d3..92d4a13d39e 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -18,7 +18,6 @@ import ( "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/services/search" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" ) @@ -238,22 +237,6 @@ func addGettingStartedPanelToHomeDashboard(dash *simplejson.Json) { row.Set("panels", panels) } -func GetDashboardFromJsonFile(c *middleware.Context) { - file := c.Params(":file") - - dashboard := search.GetDashboardFromJsonIndex(file) - if dashboard == nil { - c.JsonApiErr(404, "Dashboard not found", nil) - return - } - - dash := dtos.DashboardFullWithMeta{Dashboard: dashboard.Data} - dash.Meta.Type = m.DashTypeJson - dash.Meta.CanEdit = canEditDashboard(c.OrgRole) - - c.JSON(200, &dash) -} - // GetDashboardVersions returns all dashboard versions as JSON func GetDashboardVersions(c *middleware.Context) Response { dashboardId := c.ParamsInt64(":dashboardId") diff --git a/pkg/services/search/handlers.go b/pkg/services/search/handlers.go index a4905d6fa58..3a200389d69 100644 --- a/pkg/services/search/handlers.go +++ b/pkg/services/search/handlers.go @@ -1,38 +1,14 @@ package search import ( - "log" - "path/filepath" "sort" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/setting" ) -var jsonDashIndex *JsonDashIndex - func Init() { bus.AddHandler("search", searchHandler) - - jsonIndexCfg, _ := setting.Cfg.GetSection("dashboards.json") - - if jsonIndexCfg == nil { - log.Fatal("Config section missing: dashboards.json") - return - } - - jsonIndexEnabled := jsonIndexCfg.Key("enabled").MustBool(false) - - if jsonIndexEnabled { - jsonFilesPath := jsonIndexCfg.Key("path").String() - if !filepath.IsAbs(jsonFilesPath) { - jsonFilesPath = filepath.Join(setting.HomePath, jsonFilesPath) - } - - jsonDashIndex = NewJsonDashIndex(jsonFilesPath) - go jsonDashIndex.updateLoop() - } } func searchHandler(query *Query) error { @@ -52,15 +28,6 @@ func searchHandler(query *Query) error { hits = append(hits, dashQuery.Result...) - if jsonDashIndex != nil { - jsonHits, err := jsonDashIndex.Search(query) - if err != nil { - return err - } - - hits = append(hits, jsonHits...) - } - // filter out results with tag filter if len(query.Tags) > 0 { filtered := HitList{} @@ -126,10 +93,3 @@ func setIsStarredFlagOnSearchResults(userId int64, hits []*Hit) error { return nil } - -func GetDashboardFromJsonIndex(filename string) *m.Dashboard { - if jsonDashIndex == nil { - return nil - } - return jsonDashIndex.GetDashboard(filename) -} diff --git a/pkg/services/search/handlers_test.go b/pkg/services/search/handlers_test.go index bb355ec146f..c0ac09f9cd0 100644 --- a/pkg/services/search/handlers_test.go +++ b/pkg/services/search/handlers_test.go @@ -11,7 +11,6 @@ import ( func TestSearch(t *testing.T) { Convey("Given search query", t, func() { - jsonDashIndex = NewJsonDashIndex("../../../public/dashboards/") query := Query{Limit: 2000} bus.AddHandler("test", func(query *FindPersistedDashboardsQuery) error { diff --git a/pkg/services/search/json_index.go b/pkg/services/search/json_index.go deleted file mode 100644 index bcda432e343..00000000000 --- a/pkg/services/search/json_index.go +++ /dev/null @@ -1,140 +0,0 @@ -package search - -import ( - "os" - "path/filepath" - "strings" - "time" - - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/log" - m "github.com/grafana/grafana/pkg/models" -) - -type JsonDashIndex struct { - path string - items []*JsonDashIndexItem -} - -type JsonDashIndexItem struct { - TitleLower string - TagsCsv string - Path string - Dashboard *m.Dashboard -} - -func NewJsonDashIndex(path string) *JsonDashIndex { - log.Info("Creating json dashboard index for path: %v", path) - - index := JsonDashIndex{} - index.path = path - index.updateIndex() - return &index -} - -func (index *JsonDashIndex) updateLoop() { - ticker := time.NewTicker(time.Minute) - for { - select { - case <-ticker.C: - if err := index.updateIndex(); err != nil { - log.Error(3, "Failed to update dashboard json index %v", err) - } - } - } -} - -func (index *JsonDashIndex) Search(query *Query) ([]*Hit, error) { - results := make([]*Hit, 0) - - if query.IsStarred { - return results, nil - } - - queryStr := strings.ToLower(query.Title) - - for _, item := range index.items { - if len(results) > query.Limit { - break - } - - // add results with matchig title filter - if strings.Contains(item.TitleLower, queryStr) { - results = append(results, &Hit{ - Type: DashHitJson, - Title: item.Dashboard.Title, - Tags: item.Dashboard.GetTags(), - Uri: "file/" + item.Path, - }) - } - } - - return results, nil -} - -func (index *JsonDashIndex) GetDashboard(path string) *m.Dashboard { - for _, item := range index.items { - if item.Path == path { - return item.Dashboard - } - } - - return nil -} - -func (index *JsonDashIndex) updateIndex() error { - var items = make([]*JsonDashIndexItem, 0) - - visitor := func(path string, f os.FileInfo, err error) error { - if err != nil { - return err - } - if f.IsDir() { - if strings.HasPrefix(f.Name(), ".") { - return filepath.SkipDir - } - return nil - } - - if strings.HasSuffix(f.Name(), ".json") { - dash, err := loadDashboardFromFile(path) - if err != nil { - return err - } - - items = append(items, dash) - } - - return nil - } - - if err := filepath.Walk(index.path, visitor); err != nil { - return err - } - - index.items = items - return nil -} - -func loadDashboardFromFile(filename string) (*JsonDashIndexItem, error) { - reader, err := os.Open(filename) - if err != nil { - return nil, err - } - defer reader.Close() - - data, err := simplejson.NewFromReader(reader) - if err != nil { - return nil, err - } - - stat, _ := os.Stat(filename) - - item := &JsonDashIndexItem{} - item.Dashboard = m.NewDashboardFromJson(data) - item.TitleLower = strings.ToLower(item.Dashboard.Title) - item.TagsCsv = strings.Join(item.Dashboard.GetTags(), ",") - item.Path = stat.Name() - - return item, nil -} diff --git a/pkg/services/search/json_index_test.go b/pkg/services/search/json_index_test.go deleted file mode 100644 index 145e1ac1e99..00000000000 --- a/pkg/services/search/json_index_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package search - -import ( - "testing" - - . "github.com/smartystreets/goconvey/convey" -) - -func TestJsonDashIndex(t *testing.T) { - - Convey("Given the json dash index", t, func() { - index := NewJsonDashIndex("../../../public/dashboards/") - - Convey("Should be able to update index", func() { - err := index.updateIndex() - So(err, ShouldBeNil) - }) - - Convey("Should be able to search index", func() { - res, err := index.Search(&Query{Title: "", Limit: 20}) - So(err, ShouldBeNil) - - So(len(res), ShouldEqual, 3) - }) - - Convey("Should be able to search index by title", func() { - res, err := index.Search(&Query{Title: "home", Limit: 20}) - So(err, ShouldBeNil) - - So(len(res), ShouldEqual, 1) - So(res[0].Title, ShouldEqual, "Home") - }) - - Convey("Should not return when starred is filtered", func() { - res, err := index.Search(&Query{Title: "", IsStarred: true}) - So(err, ShouldBeNil) - - So(len(res), ShouldEqual, 0) - }) - - }) -} From f39b40c2ddf12bc1b580f60a16292b8759d92328 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 7 Dec 2017 16:29:00 +0100 Subject: [PATCH 23/44] changelog: better styling --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04dccd60276..b7e5c972716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ ## Breaking change if you are running night build -[dashboard.json] have been replaced with [dashboard provisioning](http://docs.grafana.org/administration/provisioning/). +`[dashboard.json]` have been replaced with [dashboard provisioning](http://docs.grafana.org/administration/provisioning/). Config files for provisioning datasources as configuration have changed from `/conf/datasources` to `/conf/provisioning/datasources`. From `/etc/grafana/datasources` to `/etc/grafana/provisioning/datasources` when installed with deb/rpm packages. From 7b81ebc991bb580ddb7b504e278fa73cc28596cd Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 7 Dec 2017 16:29:56 +0100 Subject: [PATCH 24/44] changelog: breaking regardless what your running --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7e5c972716..b85f8cd65a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ # 4.7.0 (unreleased) -## Breaking change if you are running night build +## Breaking changes `[dashboard.json]` have been replaced with [dashboard provisioning](http://docs.grafana.org/administration/provisioning/). From 51854ff4fadaa587747c24669ca630038c00b579 Mon Sep 17 00:00:00 2001 From: Ben Whaley Date: Thu, 7 Dec 2017 16:10:13 +0000 Subject: [PATCH 25/44] Fix go fmt --- pkg/tsdb/cloudwatch/credentials.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tsdb/cloudwatch/credentials.go b/pkg/tsdb/cloudwatch/credentials.go index 9acdf87f5db..41ceaa6676d 100644 --- a/pkg/tsdb/cloudwatch/credentials.go +++ b/pkg/tsdb/cloudwatch/credentials.go @@ -11,8 +11,8 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" "github.com/aws/aws-sdk-go/aws/credentials/endpointcreds" - "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/defaults" + "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/sts" From 79d7213e1156d0b4472c4eaa7499d443be602f70 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 8 Dec 2017 06:52:47 +0100 Subject: [PATCH 26/44] fixes issue with datasource/dash as cfg and gitignore --- .gitignore | 4 +- conf/provisioning/dashboards/custom.yaml | 6 -- conf/provisioning/datasources/custom.yaml | 92 +++++++++++------------ conf/provisioning/datasources/sample.yaml | 48 ------------ 4 files changed, 48 insertions(+), 102 deletions(-) delete mode 100644 conf/provisioning/dashboards/custom.yaml delete mode 100644 conf/provisioning/datasources/sample.yaml diff --git a/.gitignore b/.gitignore index da222a55b5b..6bba225ab2f 100644 --- a/.gitignore +++ b/.gitignore @@ -39,8 +39,8 @@ conf/custom.ini fig.yml docker-compose.yml docker-compose.yaml -/conf/dashboards/custom.yaml -/conf/datasources/custom.yaml +/conf/provisioning/dashboards/custom.yaml +/conf/provisioning/datasources/custom.yaml profile.cov /grafana .notouch diff --git a/conf/provisioning/dashboards/custom.yaml b/conf/provisioning/dashboards/custom.yaml deleted file mode 100644 index 909a621ca18..00000000000 --- a/conf/provisioning/dashboards/custom.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- name: 'default' - org_id: 1 - folder: '' - type: file - options: - folder: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/conf/provisioning/datasources/custom.yaml b/conf/provisioning/datasources/custom.yaml index 3d3e36ff108..2a02707a2ee 100644 --- a/conf/provisioning/datasources/custom.yaml +++ b/conf/provisioning/datasources/custom.yaml @@ -1,48 +1,48 @@ -# list of datasources that should be deleted from the database -delete_datasources: - - name: Graphite - org_id: 1 +# # list of datasources that should be deleted from the database +# delete_datasources: +# - name: Graphite +# org_id: 1 -# list of datasources to insert/update depending -# whats available in the datbase -datasources: - # name of the datasource. Required -- name: Graphite - # datasource type. Required - type: graphite - # access mode. direct or proxy. Required - access: proxy - # org id. will default to org_id 1 if not specified - org_id: 1 - # url - url: http://localhost:8080 - # database password, if used - password: - # database user, if used - user: - # database name, if used - database: - # enable/disable basic auth - basic_auth: - # basic auth username - basic_auth_user: - # basic auth password - basic_auth_password: - # enable/disable with credentials headers - with_credentials: - # mark as default datasource. Max one per org - is_default: - # fields that will be converted to json and stored in json_data - json_data: - graphiteVersion: "1.1" - tlsAuth: true - tlsAuthWithCACert: true - # json object of data that will be encrypted. - secure_json_data: - tlsCACert: "..." - tlsClientCert: "..." - tlsClientKey: "..." - version: 1 - # allow users to edit datasources from the UI. - editable: false +# # list of datasources to insert/update depending +# # whats available in the datbase +# datasources: +# # name of the datasource. Required +# - name: Graphite +# # datasource type. Required +# type: graphite +# # access mode. direct or proxy. Required +# access: proxy +# # org id. will default to org_id 1 if not specified +# org_id: 1 +# # url +# url: http://localhost:8080 +# # database password, if used +# password: +# # database user, if used +# user: +# # database name, if used +# database: +# # enable/disable basic auth +# basic_auth: +# # basic auth username +# basic_auth_user: +# # basic auth password +# basic_auth_password: +# # enable/disable with credentials headers +# with_credentials: +# # mark as default datasource. Max one per org +# is_default: +# # fields that will be converted to json and stored in json_data +# json_data: +# graphiteVersion: "1.1" +# tlsAuth: true +# tlsAuthWithCACert: true +# # json object of data that will be encrypted. +# secure_json_data: +# tlsCACert: "..." +# tlsClientCert: "..." +# tlsClientKey: "..." +# version: 1 +# # allow users to edit datasources from the UI. +# editable: false diff --git a/conf/provisioning/datasources/sample.yaml b/conf/provisioning/datasources/sample.yaml deleted file mode 100644 index d8ddc9c6bed..00000000000 --- a/conf/provisioning/datasources/sample.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# list of datasources that should be deleted from the database -delete_datasources: - # - name: Graphite - # org_id: 1 - -# list of datasources to insert/update depending -# whats available in the datbase -datasources: -# # name of the datasource. Required -# - name: Graphite -# # datasource type. Required -# type: graphite -# # access mode. direct or proxy. Required -# access: proxy -# # org id. will default to org_id 1 if not specified -# org_id: 1 -# # url -# url: http://localhost:8080 -# # database password, if used -# password: -# # database user, if used -# user: -# # database name, if used -# database: -# # enable/disable basic auth -# basic_auth: -# # basic auth username -# basic_auth_user: -# # basic auth password -# basic_auth_password: -# # enable/disable with credentials headers -# with_credentials: -# # mark as default datasource. Max one per org -# is_default: -# # fields that will be converted to json and stored in json_data -# json_data: -# graphiteVersion: "1.1" -# tlsAuth: true -# tlsAuthWithCACert: true -# # json object of data that will be encrypted. -# secure_json_data: -# tlsCACert: "..." -# tlsClientCert: "..." -# tlsClientKey: "..." -# version: 1 -# # allow users to edit datasources from the UI. -# editable: false - From 80a852b00a2bbe0c4fc57d9e6b15e7c3ef33f28a Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 8 Dec 2017 09:35:46 +0100 Subject: [PATCH 27/44] make gitignore more generic --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6bba225ab2f..31c89aadf6b 100644 --- a/.gitignore +++ b/.gitignore @@ -39,8 +39,7 @@ conf/custom.ini fig.yml docker-compose.yml docker-compose.yaml -/conf/provisioning/dashboards/custom.yaml -/conf/provisioning/datasources/custom.yaml +/conf/provisioning/**/custom.yaml profile.cov /grafana .notouch From c7668023253130353c00204ccf3163e99757d7e3 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 8 Dec 2017 10:28:38 +0100 Subject: [PATCH 28/44] improve sample datasource.yaml --- conf/provisioning/datasources/custom.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/provisioning/datasources/custom.yaml b/conf/provisioning/datasources/custom.yaml index 2a02707a2ee..7ed3a8430c6 100644 --- a/conf/provisioning/datasources/custom.yaml +++ b/conf/provisioning/datasources/custom.yaml @@ -1,11 +1,11 @@ # # list of datasources that should be deleted from the database -# delete_datasources: +delete_datasources: # - name: Graphite # org_id: 1 # # list of datasources to insert/update depending # # whats available in the datbase -# datasources: +datasources: # # name of the datasource. Required # - name: Graphite # # datasource type. Required From 5f5cdad97aa8227255229157ac41907f637e1956 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 8 Dec 2017 10:50:11 +0100 Subject: [PATCH 29/44] improve error handling for datasources as cfg --- .../datasources/{custom.yaml => sample.yaml} | 4 +- ...ashboard_test.go => config_reader_test.go} | 13 +++++ .../broken-configs/commented.yaml | 6 +++ .../provisioning/datasources/datasources.go | 14 ++++-- .../test-configs/broken-yaml/commented.yaml | 48 +++++++++++++++++++ 5 files changed, 79 insertions(+), 6 deletions(-) rename conf/provisioning/datasources/{custom.yaml => sample.yaml} (97%) rename pkg/services/provisioning/dashboards/{dashboard_test.go => config_reader_test.go} (78%) create mode 100644 pkg/services/provisioning/dashboards/test-configs/broken-configs/commented.yaml create mode 100644 pkg/services/provisioning/datasources/test-configs/broken-yaml/commented.yaml diff --git a/conf/provisioning/datasources/custom.yaml b/conf/provisioning/datasources/sample.yaml similarity index 97% rename from conf/provisioning/datasources/custom.yaml rename to conf/provisioning/datasources/sample.yaml index 7ed3a8430c6..1bb9cb53b45 100644 --- a/conf/provisioning/datasources/custom.yaml +++ b/conf/provisioning/datasources/sample.yaml @@ -1,11 +1,11 @@ # # list of datasources that should be deleted from the database -delete_datasources: +#delete_datasources: # - name: Graphite # org_id: 1 # # list of datasources to insert/update depending # # whats available in the datbase -datasources: +#datasources: # # name of the datasource. Required # - name: Graphite # # datasource type. Required diff --git a/pkg/services/provisioning/dashboards/dashboard_test.go b/pkg/services/provisioning/dashboards/config_reader_test.go similarity index 78% rename from pkg/services/provisioning/dashboards/dashboard_test.go rename to pkg/services/provisioning/dashboards/config_reader_test.go index e9ab352a122..657c0045e3a 100644 --- a/pkg/services/provisioning/dashboards/dashboard_test.go +++ b/pkg/services/provisioning/dashboards/config_reader_test.go @@ -8,6 +8,7 @@ import ( var ( simpleDashboardConfig string = "./test-configs/dashboards-from-disk" + brokenConfigs string = "./test-configs/borken-configs" ) func TestDashboardsAsConfig(t *testing.T) { @@ -45,5 +46,17 @@ func TestDashboardsAsConfig(t *testing.T) { So(len(ds2.Options), ShouldEqual, 1) So(ds2.Options["folder"], ShouldEqual, "/var/lib/grafana/dashboards") }) + + Convey("Should skip broken config files", func() { + + cfgProvifer := configReader{path: brokenConfigs} + cfg, err := cfgProvifer.readConfig() + if err != nil { + t.Fatalf("readConfig return an error %v", err) + } + + So(len(cfg), ShouldEqual, 0) + + }) }) } diff --git a/pkg/services/provisioning/dashboards/test-configs/broken-configs/commented.yaml b/pkg/services/provisioning/dashboards/test-configs/broken-configs/commented.yaml new file mode 100644 index 00000000000..e40612af508 --- /dev/null +++ b/pkg/services/provisioning/dashboards/test-configs/broken-configs/commented.yaml @@ -0,0 +1,6 @@ +# - name: 'default' +# org_id: 1 +# folder: '' +# type: file +# options: +# folder: /var/lib/grafana/dashboards diff --git a/pkg/services/provisioning/datasources/datasources.go b/pkg/services/provisioning/datasources/datasources.go index 325dbbbd757..ce631c565d4 100644 --- a/pkg/services/provisioning/datasources/datasources.go +++ b/pkg/services/provisioning/datasources/datasources.go @@ -118,13 +118,19 @@ func (configReader) readConfig(path string) ([]*DatasourcesAsConfig, error) { return nil, err } - datasources = append(datasources, datasource) + if datasource != nil { + datasources = append(datasources, datasource) + } } } defaultCount := 0 - for _, cfg := range datasources { - for _, ds := range cfg.Datasources { + for i := range datasources { + if datasources[i].Datasources == nil { + continue + } + + for _, ds := range datasources[i].Datasources { if ds.OrgId == 0 { ds.OrgId = 1 } @@ -137,7 +143,7 @@ func (configReader) readConfig(path string) ([]*DatasourcesAsConfig, error) { } } - for _, ds := range cfg.DeleteDatasources { + for _, ds := range datasources[i].DeleteDatasources { if ds.OrgId == 0 { ds.OrgId = 1 } diff --git a/pkg/services/provisioning/datasources/test-configs/broken-yaml/commented.yaml b/pkg/services/provisioning/datasources/test-configs/broken-yaml/commented.yaml new file mode 100644 index 00000000000..1bb9cb53b45 --- /dev/null +++ b/pkg/services/provisioning/datasources/test-configs/broken-yaml/commented.yaml @@ -0,0 +1,48 @@ +# # list of datasources that should be deleted from the database +#delete_datasources: +# - name: Graphite +# org_id: 1 + +# # list of datasources to insert/update depending +# # whats available in the datbase +#datasources: +# # name of the datasource. Required +# - name: Graphite +# # datasource type. Required +# type: graphite +# # access mode. direct or proxy. Required +# access: proxy +# # org id. will default to org_id 1 if not specified +# org_id: 1 +# # url +# url: http://localhost:8080 +# # database password, if used +# password: +# # database user, if used +# user: +# # database name, if used +# database: +# # enable/disable basic auth +# basic_auth: +# # basic auth username +# basic_auth_user: +# # basic auth password +# basic_auth_password: +# # enable/disable with credentials headers +# with_credentials: +# # mark as default datasource. Max one per org +# is_default: +# # fields that will be converted to json and stored in json_data +# json_data: +# graphiteVersion: "1.1" +# tlsAuth: true +# tlsAuthWithCACert: true +# # json object of data that will be encrypted. +# secure_json_data: +# tlsCACert: "..." +# tlsClientCert: "..." +# tlsClientKey: "..." +# version: 1 +# # allow users to edit datasources from the UI. +# editable: false + From 0c5ef1453df49ad332a0bbe4995de95328f28be3 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 8 Dec 2017 11:33:15 +0100 Subject: [PATCH 30/44] fixes broken test --- pkg/services/provisioning/dashboards/config_reader_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/provisioning/dashboards/config_reader_test.go b/pkg/services/provisioning/dashboards/config_reader_test.go index 657c0045e3a..56c5a5fcf3d 100644 --- a/pkg/services/provisioning/dashboards/config_reader_test.go +++ b/pkg/services/provisioning/dashboards/config_reader_test.go @@ -8,7 +8,7 @@ import ( var ( simpleDashboardConfig string = "./test-configs/dashboards-from-disk" - brokenConfigs string = "./test-configs/borken-configs" + brokenConfigs string = "./test-configs/broken-configs" ) func TestDashboardsAsConfig(t *testing.T) { From ce809de1ed98841b4414afeab27e2c8e736a9faa Mon Sep 17 00:00:00 2001 From: Sven Klemm <31455525+svenklemm@users.noreply.github.com> Date: Fri, 8 Dec 2017 15:14:10 +0100 Subject: [PATCH 31/44] postgres: change $__timeGroup macro to include "AS time" column alias (#10119) * change $__timeGroup macro to include column alias * update docs and help text for $__timeGroup macro --- docs/sources/features/datasources/postgres.md | 6 +++--- pkg/tsdb/postgres/macros.go | 2 +- pkg/tsdb/postgres/macros_test.go | 2 +- .../plugins/datasource/postgres/partials/query.editor.html | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/sources/features/datasources/postgres.md b/docs/sources/features/datasources/postgres.md index e9d65b8f327..7d52df2fd3e 100644 --- a/docs/sources/features/datasources/postgres.md +++ b/docs/sources/features/datasources/postgres.md @@ -48,7 +48,7 @@ Macro example | Description *$__timeFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name. For example, *extract(epoch from dateColumn) BETWEEN 1494410783 AND 1494497183* *$__timeFrom()* | Will be replaced by the start of the currently active time selection. For example, *to_timestamp(1494410783)* *$__timeTo()* | Will be replaced by the end of the currently active time selection. For example, *to_timestamp(1494497183)* -*$__timeGroup(dateColumn,'5m')* | Will be replaced by an expression usable in GROUP BY clause. For example, *(extract(epoch from "dateColumn")/300)::bigint*300* +*$__timeGroup(dateColumn,'5m')* | Will be replaced by an expression usable in GROUP BY clause. For example, *(extract(epoch from dateColumn)/300)::bigint*300 AS time* *$__unixEpochFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name with times represented as unix timestamp. For example, *dateColumn > 1494410783 AND dateColumn < 1494497183* *$__unixEpochFrom()* | Will be replaced by the start of the currently active time selection as unix timestamp. For example, *1494410783* *$__unixEpochTo()* | Will be replaced by the end of the currently active time selection as unix timestamp. For example, *1494497183* @@ -94,7 +94,7 @@ Example with `metric` column ```sql SELECT - $__timeGroup(time_date_time,'5m') as time, + $__timeGroup(time_date_time,'5m'), min(value_double), 'min' as metric FROM test_data @@ -107,7 +107,7 @@ Example with multiple columns: ```sql SELECT - $__timeGroup(time_date_time,'5m') as time, + $__timeGroup(time_date_time,'5m'), min(value_double) as min_value, max(value_double) as max_value FROM test_data diff --git a/pkg/tsdb/postgres/macros.go b/pkg/tsdb/postgres/macros.go index 288787589ce..086eb96655f 100644 --- a/pkg/tsdb/postgres/macros.go +++ b/pkg/tsdb/postgres/macros.go @@ -89,7 +89,7 @@ func (m *PostgresMacroEngine) evaluateMacro(name string, args []string) (string, if err != nil { return "", fmt.Errorf("error parsing interval %v", args[1]) } - return fmt.Sprintf("(extract(epoch from \"%s\")/%v)::bigint*%v", args[0], interval.Seconds(), interval.Seconds()), nil + return fmt.Sprintf("(extract(epoch from %s)/%v)::bigint*%v AS time", args[0], interval.Seconds(), interval.Seconds()), nil case "__unixEpochFilter": if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) diff --git a/pkg/tsdb/postgres/macros_test.go b/pkg/tsdb/postgres/macros_test.go index ff268805259..ebc5191d46e 100644 --- a/pkg/tsdb/postgres/macros_test.go +++ b/pkg/tsdb/postgres/macros_test.go @@ -45,7 +45,7 @@ func TestMacroEngine(t *testing.T) { sql, err := engine.Interpolate(timeRange, "GROUP BY $__timeGroup(time_column,'5m')") So(err, ShouldBeNil) - So(sql, ShouldEqual, "GROUP BY (extract(epoch from \"time_column\")/300)::bigint*300") + So(sql, ShouldEqual, "GROUP BY (extract(epoch from time_column)/300)::bigint*300 AS time") }) Convey("interpolate __timeTo function", func() { diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 574fca33901..163970a9ad5 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -50,11 +50,11 @@ Macros: - $__timeEpoch -> extract(epoch from column) as "time" - $__timeFilter(column) -> extract(epoch from column) BETWEEN 1492750877 AND 1492750877 - $__unixEpochFilter(column) -> column > 1492750877 AND column < 1492750877 -- $__timeGroup(column,'5m') -> (extract(epoch from "dateColumn")/300)::bigint*300 +- $__timeGroup(column,'5m') -> (extract(epoch from column)/300)::bigint*300 AS time Example of group by and order by with $__timeGroup: SELECT - $__timeGroup(date_time_col, '1h') AS time, + $__timeGroup(date_time_col, '1h'), sum(value) as value FROM yourtable GROUP BY time From 0506cfdc1d767496df9aeec8c10400be381bd5e0 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 8 Dec 2017 16:09:32 +0100 Subject: [PATCH 32/44] changelog: adds ntoe about closing #10111 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b85f8cd65a6..b8b1dc8bdce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,9 +33,12 @@ From `/etc/grafana/datasources` to `/etc/grafana/provisioning/datasources` when * **Dashboard**: Make it possible to start dashboards from search and dashboard list panel [#1871](https://github.com/grafana/grafana/issues/1871) * **Annotations**: Posting annotations now return the id of the annotation [#9798](https://github.com/grafana/grafana/issues/9798) * **Systemd**: Use systemd notification ready flag [#10024](https://github.com/grafana/grafana/issues/10024), thx [@jgrassler](https://github.com/jgrassler) +* **Github**: Use organizations_url provided from github to verify user belongs in org. [#10111](https://github.com/grafana/grafana/issues/10111), thx [@adiletmaratov](https://github.com/adiletmaratov) + ## Tech * **RabbitMq**: Remove support for publishing events to RabbitMQ [#9645](https://github.com/grafana/grafana/issues/9645) + ## Fixes * **Sensu**: Send alert message to sensu output [#9551](https://github.com/grafana/grafana/issues/9551), thx [@cjchand](https://github.com/cjchand) * **Singlestat**: suppress error when result contains no datapoints [#9636](https://github.com/grafana/grafana/issues/9636), thx [@utkarshcmu](https://github.com/utkarshcmu) From 4fcf79a3a6c358608b30bea159e2d6f2c29218cb Mon Sep 17 00:00:00 2001 From: Dan Cech Date: Sat, 9 Dec 2017 11:17:34 -0500 Subject: [PATCH 33/44] demonstrate parseTarget issue --- .../plugins/datasource/graphite/specs/query_ctrl_specs.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/app/plugins/datasource/graphite/specs/query_ctrl_specs.ts b/public/app/plugins/datasource/graphite/specs/query_ctrl_specs.ts index 177c1e2a0d6..22d93976043 100644 --- a/public/app/plugins/datasource/graphite/specs/query_ctrl_specs.ts +++ b/public/app/plugins/datasource/graphite/specs/query_ctrl_specs.ts @@ -170,7 +170,7 @@ describe('GraphiteQueryCtrl', function() { describe('when updating targets with nested query', function() { beforeEach(function() { - ctx.ctrl.target.target = 'scaleToSeconds(#A)'; + ctx.ctrl.target.target = 'scaleToSeconds(#A, 60)'; ctx.ctrl.datasource.metricFindQuery = sinon.stub().returns(ctx.$q.when([{expandable: false}])); ctx.ctrl.parseTarget(); @@ -183,11 +183,11 @@ describe('GraphiteQueryCtrl', function() { }); it('target should remain the same', function() { - expect(ctx.ctrl.target.target).to.be('scaleToSeconds(#A)'); + expect(ctx.ctrl.target.target).to.be('scaleToSeconds(#A, 60)'); }); it('targetFull should include nexted queries', function() { - expect(ctx.ctrl.target.targetFull).to.be('scaleToSeconds(nested.query.count)'); + expect(ctx.ctrl.target.targetFull).to.be('scaleToSeconds(nested.query.count, 60)'); }); }); From 3a1700cbeea2ad6226094992590057f1b9ab166b Mon Sep 17 00:00:00 2001 From: Dan Cech Date: Sat, 9 Dec 2017 16:27:05 -0500 Subject: [PATCH 34/44] improve handling of query references --- .../datasource/graphite/graphite_query.ts | 23 ++++++++++--------- .../plugins/datasource/graphite/query_ctrl.ts | 21 ++++++++++++++++- .../graphite/specs/query_ctrl_specs.ts | 6 ++--- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/public/app/plugins/datasource/graphite/graphite_query.ts b/public/app/plugins/datasource/graphite/graphite_query.ts index 2a70667fdac..c97cf725459 100644 --- a/public/app/plugins/datasource/graphite/graphite_query.ts +++ b/public/app/plugins/datasource/graphite/graphite_query.ts @@ -91,27 +91,25 @@ export default class GraphiteQuery { this.functions.push(innerFunc); break; case 'series-ref': - this.addFunctionParameter(func, astNode.value, index, this.segments.length > 0); + if (this.segments.length > 0) { + this.addFunctionParameter(func, astNode.value, index, true); + } else { + this.segments.push(astNode); + } break; case 'bool': case 'string': case 'number': - if ((index-1) >= func.def.params.length) { - throw { message: 'invalid number of parameters to method ' + func.def.name }; - } var shiftBack = this.isShiftParamsBack(func); this.addFunctionParameter(func, astNode.value, index, shiftBack); - break; + break; case 'metric': if (this.segments.length > 0) { - if (astNode.segments.length !== 1) { - throw { message: 'Multiple metric params not supported, use text editor.' }; + this.addFunctionParameter(func, _.join(_.map(astNode.segments, 'value'), '.'), index, true); + } else { + this.segments = astNode.segments; } - this.addFunctionParameter(func, astNode.segments[0].value, index, true); break; - } - - this.segments = astNode.segments; } } @@ -149,6 +147,9 @@ export default class GraphiteQuery { if (shiftBack) { index = Math.max(index - 1, 0); } + if (index > func.def.params.length) { + throw { message: 'too many parameters for function ' + func.def.name }; + } func.params[index] = value; } diff --git a/public/app/plugins/datasource/graphite/query_ctrl.ts b/public/app/plugins/datasource/graphite/query_ctrl.ts index c14836b4bdd..f5daf01f7cc 100644 --- a/public/app/plugins/datasource/graphite/query_ctrl.ts +++ b/public/app/plugins/datasource/graphite/query_ctrl.ts @@ -62,6 +62,10 @@ export class GraphiteQueryCtrl extends QueryCtrl { } checkOtherSegments(fromIndex) { + if (this.queryModel.segments.length === 1 && this.queryModel.segments[0].type === 'series-ref') { + return; + } + if (fromIndex === 0) { this.addSelectMetricSegment(); return; @@ -108,8 +112,23 @@ export class GraphiteQueryCtrl extends QueryCtrl { if (altSegments.length === 0) { return altSegments; } + // add query references + if (index === 0) { + _.eachRight(this.panelCtrl.panel.targets, target => { + if (target.refId === this.queryModel.target.refId) { + return; + } + + altSegments.unshift(this.uiSegmentSrv.newSegment({ + type: 'series-ref', + value: '#' + target.refId, + expandable: false, + })); + }); + } + // add template variables - _.each(this.templateSrv.variables, variable => { + _.eachRight(this.templateSrv.variables, variable => { altSegments.unshift(this.uiSegmentSrv.newSegment({ type: 'template', value: '$' + variable.name, diff --git a/public/app/plugins/datasource/graphite/specs/query_ctrl_specs.ts b/public/app/plugins/datasource/graphite/specs/query_ctrl_specs.ts index 22d93976043..86c32c3c018 100644 --- a/public/app/plugins/datasource/graphite/specs/query_ctrl_specs.ts +++ b/public/app/plugins/datasource/graphite/specs/query_ctrl_specs.ts @@ -95,11 +95,11 @@ describe('GraphiteQueryCtrl', function() { }); it('should not add select metric segment', function() { - expect(ctx.ctrl.segments.length).to.be(0); + expect(ctx.ctrl.segments.length).to.be(1); }); - it('should add both series refs as params', function() { - expect(ctx.ctrl.queryModel.functions[0].params.length).to.be(2); + it('should add second series ref as param', function() { + expect(ctx.ctrl.queryModel.functions[0].params.length).to.be(1); }); }); From 8dcfa18761d9eddc9b50aed680506ef1b35008e8 Mon Sep 17 00:00:00 2001 From: Dan Cech Date: Sun, 10 Dec 2017 10:45:41 -0500 Subject: [PATCH 35/44] simplify function parameter addition --- .../datasource/graphite/graphite_query.ts | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/public/app/plugins/datasource/graphite/graphite_query.ts b/public/app/plugins/datasource/graphite/graphite_query.ts index c97cf725459..3b29c57b808 100644 --- a/public/app/plugins/datasource/graphite/graphite_query.ts +++ b/public/app/plugins/datasource/graphite/graphite_query.ts @@ -46,7 +46,7 @@ export default class GraphiteQuery { } try { - this.parseTargetRecursive(astNode, null, 0); + this.parseTargetRecursive(astNode, null); } catch (err) { console.log('error parsing target:', err.message); this.error = err.message; @@ -75,7 +75,7 @@ export default class GraphiteQuery { }, ""); } - parseTargetRecursive(astNode, func, index) { + parseTargetRecursive(astNode, func) { if (astNode === null) { return null; } @@ -83,8 +83,8 @@ export default class GraphiteQuery { switch (astNode.type) { case 'function': var innerFunc = gfunc.createFuncInstance(astNode.name, { withDefaultParams: false }); - _.each(astNode.params, (param, index) => { - this.parseTargetRecursive(param, innerFunc, index); + _.each(astNode.params, param => { + this.parseTargetRecursive(param, innerFunc); }); innerFunc.updateText(); @@ -92,7 +92,7 @@ export default class GraphiteQuery { break; case 'series-ref': if (this.segments.length > 0) { - this.addFunctionParameter(func, astNode.value, index, true); + this.addFunctionParameter(func, astNode.value); } else { this.segments.push(astNode); } @@ -100,12 +100,11 @@ export default class GraphiteQuery { case 'bool': case 'string': case 'number': - var shiftBack = this.isShiftParamsBack(func); - this.addFunctionParameter(func, astNode.value, index, shiftBack); + this.addFunctionParameter(func, astNode.value); break; case 'metric': if (this.segments.length > 0) { - this.addFunctionParameter(func, _.join(_.map(astNode.segments, 'value'), '.'), index, true); + this.addFunctionParameter(func, _.join(_.map(astNode.segments, 'value'), '.')); } else { this.segments = astNode.segments; } @@ -113,10 +112,6 @@ export default class GraphiteQuery { } } - isShiftParamsBack(func) { - return func.def.name !== 'seriesByTag'; - } - updateSegmentValue(segment, index) { this.segments[index].value = segment.value; } @@ -143,14 +138,11 @@ export default class GraphiteQuery { } } - addFunctionParameter(func, value, index, shiftBack) { - if (shiftBack) { - index = Math.max(index - 1, 0); - } - if (index > func.def.params.length) { + addFunctionParameter(func, value) { + if (func.params.length >= func.def.params.length) { throw { message: 'too many parameters for function ' + func.def.name }; } - func.params[index] = value; + func.params.push(value); } removeFunction(func) { From 525ae4fdf5a1986401a775ea2f3641ec7cc8f39e Mon Sep 17 00:00:00 2001 From: Dan Cech Date: Sun, 10 Dec 2017 10:46:31 -0500 Subject: [PATCH 36/44] support metric trees of varying depth, never send '.select metric' to graphite --- public/app/plugins/datasource/graphite/graphite_query.ts | 2 +- public/app/plugins/datasource/graphite/query_ctrl.ts | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/public/app/plugins/datasource/graphite/graphite_query.ts b/public/app/plugins/datasource/graphite/graphite_query.ts index 3b29c57b808..54504219dad 100644 --- a/public/app/plugins/datasource/graphite/graphite_query.ts +++ b/public/app/plugins/datasource/graphite/graphite_query.ts @@ -152,7 +152,7 @@ export default class GraphiteQuery { updateModelTarget(targets) { // render query if (!this.target.textEditor) { - var metricPath = this.getSegmentPathUpTo(this.segments.length); + var metricPath = this.getSegmentPathUpTo(this.segments.length).replace(/\.select metric$/, ''); this.target.target = _.reduce(this.functions, wrapFunction, metricPath); } diff --git a/public/app/plugins/datasource/graphite/query_ctrl.ts b/public/app/plugins/datasource/graphite/query_ctrl.ts index f5daf01f7cc..d00ac90fe65 100644 --- a/public/app/plugins/datasource/graphite/query_ctrl.ts +++ b/public/app/plugins/datasource/graphite/query_ctrl.ts @@ -219,10 +219,7 @@ export class GraphiteQueryCtrl extends QueryCtrl { this.updateModelTarget(); if (this.queryModel.target !== oldTarget) { - var lastSegment = this.segments.length > 0 ? this.segments[this.segments.length - 1] : {}; - if (lastSegment.value !== 'select metric') { - this.panelCtrl.refresh(); - } + this.panelCtrl.refresh(); } } From e6bdccbb05bf9b0b8e8224796ee3bc75f3beb874 Mon Sep 17 00:00:00 2001 From: noakup Date: Sun, 10 Dec 2017 18:38:48 +0200 Subject: [PATCH 37/44] hyphenhyphen --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index aefc0c0802b..5358cd3f3d5 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ If you have any problems please read the [troubleshooting guide](http://docs.gra Be sure to read the [getting started guide](http://docs.grafana.org/guides/gettingstarted/) and the other feature guides. ## Run from master -If you want to build a package yourself, or contribute. Here is a guide for how to do that. You can always find +If you want to build a package yourself, or contribute - Here is a guide for how to do that. You can always find the latest master builds [here](https://grafana.com/grafana/download) ### Dependencies @@ -97,7 +97,7 @@ Writing & watching frontend tests (we have two test runners) ## Contribute -If you have any idea for an improvement or found a bug do not hesitate to open an issue. +If you have any idea for an improvement or found a bug, do not hesitate to open an issue. And if you have time clone this repo and submit a pull request and help me make Grafana the kickass metrics & devops dashboard we all dream about! From 588ce6606c4c0a3873539b713ba2ec2fd3b05e79 Mon Sep 17 00:00:00 2001 From: Dan Cech Date: Sun, 10 Dec 2017 21:20:38 -0500 Subject: [PATCH 38/44] fix function re-ordering broken in #9436 --- public/app/plugins/datasource/graphite/func_editor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/app/plugins/datasource/graphite/func_editor.js b/public/app/plugins/datasource/graphite/func_editor.js index 57cdc844375..534b0886fff 100644 --- a/public/app/plugins/datasource/graphite/func_editor.js +++ b/public/app/plugins/datasource/graphite/func_editor.js @@ -208,7 +208,7 @@ function (angular, _, $) { if ($target.hasClass('fa-arrow-left')) { $scope.$apply(function() { - _.move(ctrl.functions, $scope.$index, $scope.$index - 1); + _.move(ctrl.queryModel.functions, $scope.$index, $scope.$index - 1); ctrl.targetChanged(); }); return; @@ -216,7 +216,7 @@ function (angular, _, $) { if ($target.hasClass('fa-arrow-right')) { $scope.$apply(function() { - _.move(ctrl.functions, $scope.$index, $scope.$index + 1); + _.move(ctrl.queryModel.functions, $scope.$index, $scope.$index + 1); ctrl.targetChanged(); }); return; From f7ed24475cf32a77388620544062d380c74d206d Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 8 Dec 2017 14:11:48 +0100 Subject: [PATCH 39/44] wait for all sub routines to finish simple solution for waiting for all go sub routines to finish before closing Grafana. We would use errGroup here as well but I dont like spreading context's all over the place. closes #10131 --- pkg/api/http_server.go | 2 +- pkg/cmd/grafana-server/main.go | 33 ++++++++++++++++++------- pkg/cmd/grafana-server/server.go | 42 ++++++++++++-------------------- pkg/models/server.go | 6 ----- 4 files changed, 41 insertions(+), 42 deletions(-) delete mode 100644 pkg/models/server.go diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index 89456d20d8c..0366b9aedad 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -95,7 +95,7 @@ func (hs *HttpServer) Start(ctx context.Context) error { func (hs *HttpServer) Shutdown(ctx context.Context) error { err := hs.httpSrv.Shutdown(ctx) - hs.log.Info("stopped http server") + hs.log.Info("Stopped HTTP server") return err } diff --git a/pkg/cmd/grafana-server/main.go b/pkg/cmd/grafana-server/main.go index 183e4b047cd..edf827ac4ad 100644 --- a/pkg/cmd/grafana-server/main.go +++ b/pkg/cmd/grafana-server/main.go @@ -14,8 +14,8 @@ import ( "net/http" _ "net/http/pprof" + "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/metrics" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" _ "github.com/grafana/grafana/pkg/services/alerting/conditions" @@ -40,9 +40,6 @@ var homePath = flag.String("homepath", "", "path to grafana install/home path, d var pidFile = flag.String("pidfile", "", "path to pid file") var exitChan = make(chan int) -func init() { -} - func main() { v := flag.Bool("v", false, "prints current version and exits") profile := flag.Bool("profile", false, "Turn on pprof profiling") @@ -82,12 +79,28 @@ func main() { setting.BuildStamp = buildstampInt64 metrics.M_Grafana_Version.WithLabelValues(version).Set(1) - + shutdownCompleted := make(chan int) server := NewGrafanaServer() - server.Start() + + go listenToSystemSignals(server, shutdownCompleted) + + go func() { + code := 0 + if err := server.Start(); err != nil { + log.Error2("Startup failed", "error", err) + code = 1 + } + + exitChan <- code + }() + + code := <-shutdownCompleted + log.Info2("Grafana shutdown completed.", "code", code) + log.Close() + os.Exit(code) } -func listenToSystemSignals(server models.GrafanaServer) { +func listenToSystemSignals(server *GrafanaServerImpl, shutdownCompleted chan int) { signalChan := make(chan os.Signal, 1) ignoreChan := make(chan os.Signal, 1) code := 0 @@ -97,10 +110,12 @@ func listenToSystemSignals(server models.GrafanaServer) { select { case sig := <-signalChan: - // Stops trace if profiling has been enabled - trace.Stop() + trace.Stop() // Stops trace if profiling has been enabled server.Shutdown(0, fmt.Sprintf("system signal: %s", sig)) + shutdownCompleted <- 0 case code = <-exitChan: + trace.Stop() // Stops trace if profiling has been enabled server.Shutdown(code, "startup error") + shutdownCompleted <- code } } diff --git a/pkg/cmd/grafana-server/server.go b/pkg/cmd/grafana-server/server.go index 820295a9b88..b84c3d4e3d6 100644 --- a/pkg/cmd/grafana-server/server.go +++ b/pkg/cmd/grafana-server/server.go @@ -11,7 +11,6 @@ import ( "strconv" "time" - "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/services/provisioning" "golang.org/x/sync/errgroup" @@ -20,7 +19,6 @@ import ( "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/login" "github.com/grafana/grafana/pkg/metrics" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/cleanup" @@ -33,7 +31,7 @@ import ( "github.com/grafana/grafana/pkg/tracing" ) -func NewGrafanaServer() models.GrafanaServer { +func NewGrafanaServer() *GrafanaServerImpl { rootCtx, shutdownFn := context.WithCancel(context.Background()) childRoutines, childCtx := errgroup.WithContext(rootCtx) @@ -54,9 +52,7 @@ type GrafanaServerImpl struct { httpServer *api.HttpServer } -func (g *GrafanaServerImpl) Start() { - go listenToSystemSignals(g) - +func (g *GrafanaServerImpl) Start() error { g.initLogging() g.writePIDFile() @@ -69,16 +65,12 @@ func (g *GrafanaServerImpl) Start() { plugins.Init() if err := provisioning.Init(g.context, setting.HomePath, setting.Cfg); err != nil { - logger.Error("Failed to provision Grafana from config", "error", err) - g.Shutdown(1, "Startup failed") - return + return fmt.Errorf("Failed to provision Grafana from config. error: %v", err) } closer, err := tracing.Init(setting.Cfg) if err != nil { - g.log.Error("Tracing settings is not valid", "error", err) - g.Shutdown(1, "Startup failed") - return + return fmt.Errorf("Tracing settings is not valid. error: %v", err) } defer closer.Close() @@ -93,13 +85,12 @@ func (g *GrafanaServerImpl) Start() { g.childRoutines.Go(func() error { return cleanUpService.Run(g.context) }) if err = notifications.Init(); err != nil { - g.log.Error("Notification service failed to initialize", "error", err) - g.Shutdown(1, "Startup failed") - return + return fmt.Errorf("Notification service failed to initialize. error: %v", err) } - SendSystemdNotification("READY=1") - g.startHttpServer() + sendSystemdNotification("READY=1") + + return g.startHttpServer() } func initSql() { @@ -123,16 +114,16 @@ func (g *GrafanaServerImpl) initLogging() { setting.LogConfigurationInfo() } -func (g *GrafanaServerImpl) startHttpServer() { +func (g *GrafanaServerImpl) startHttpServer() error { g.httpServer = api.NewHttpServer() err := g.httpServer.Start(g.context) if err != nil { - g.log.Error("Fail to start server", "error", err) - g.Shutdown(1, "Startup failed") - return + return fmt.Errorf("Fail to start server. error: %v", err) } + + return nil } func (g *GrafanaServerImpl) Shutdown(code int, reason string) { @@ -145,10 +136,9 @@ func (g *GrafanaServerImpl) Shutdown(code int, reason string) { g.shutdownFn() err = g.childRoutines.Wait() - - g.log.Info("Shutdown completed", "reason", err) - log.Close() - os.Exit(code) + if err != nil && err != context.Canceled { + g.log.Error("Server shutdown completed with an error", "error", err) + } } func (g *GrafanaServerImpl) writePIDFile() { @@ -173,7 +163,7 @@ func (g *GrafanaServerImpl) writePIDFile() { g.log.Info("Writing PID file", "path", *pidFile, "pid", pid) } -func SendSystemdNotification(state string) error { +func sendSystemdNotification(state string) error { notifySocket := os.Getenv("NOTIFY_SOCKET") if notifySocket == "" { diff --git a/pkg/models/server.go b/pkg/models/server.go deleted file mode 100644 index 4d683835256..00000000000 --- a/pkg/models/server.go +++ /dev/null @@ -1,6 +0,0 @@ -package models - -type GrafanaServer interface { - Start() - Shutdown(code int, reason string) -} From ad2a69bff4e8dfd3933b339783d1da15ce30c1fb Mon Sep 17 00:00:00 2001 From: jomenxiao Date: Mon, 11 Dec 2017 16:01:17 +0800 Subject: [PATCH 40/44] add encoding param --- pkg/api/render.go | 1 + pkg/components/renderer/renderer.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/pkg/api/render.go b/pkg/api/render.go index cab9c81505d..e9bce5bc99b 100644 --- a/pkg/api/render.go +++ b/pkg/api/render.go @@ -24,6 +24,7 @@ func RenderToPng(c *middleware.Context) { OrgId: c.OrgId, Timeout: queryReader.Get("timeout", "60"), Timezone: queryReader.Get("tz", ""), + Encoding: queryReader.Get("encoding", ""), } pngPath, err := renderer.RenderToPng(renderOpts) diff --git a/pkg/components/renderer/renderer.go b/pkg/components/renderer/renderer.go index d5980231f0e..fedb68f2e31 100644 --- a/pkg/components/renderer/renderer.go +++ b/pkg/components/renderer/renderer.go @@ -27,6 +27,7 @@ type RenderOpts struct { Timeout string OrgId int64 Timezone string + Encoding string } var ErrTimeout = errors.New("Timeout error. You can set timeout in seconds with &timeout url parameter") @@ -95,6 +96,10 @@ func RenderToPng(params *RenderOpts) (string, error) { "renderKey=" + renderKey, } + if params.Encoding != "" { + cmdArgs = append([]string{fmt.Sprintf("--output-encoding=%s", params.Encoding)}, cmdArgs...) + } + cmd := exec.Command(binPath, cmdArgs...) stdout, err := cmd.StdoutPipe() From 8e7166b5c4a77c52d68dc0c01adc67eeb8b2a54c Mon Sep 17 00:00:00 2001 From: Mikael Olenfalk Date: Mon, 11 Dec 2017 09:37:27 +0100 Subject: [PATCH 41/44] Explicitly specify default region in CloudWatch datasource (#9440) The datasource uses the default region in the query if the region is "" in the settings. However setting the region to an empty string is almost impossible and rendered incorrectly. This commit introduces a special value "default" for region which is shown in the drop down and is translated to the default region of the data source when performing queries. --- .../features/datasources/cloudwatch.md | 5 +- pkg/tsdb/cloudwatch/credentials.go | 5 ++ .../datasource/cloudwatch/datasource.js | 21 ++++-- .../cloudwatch/query_parameter_ctrl.ts | 8 ++- .../cloudwatch/specs/datasource_specs.ts | 69 +++++++++++++++++++ 5 files changed, 98 insertions(+), 10 deletions(-) diff --git a/docs/sources/features/datasources/cloudwatch.md b/docs/sources/features/datasources/cloudwatch.md index bdf661dc4fc..648957ed96e 100644 --- a/docs/sources/features/datasources/cloudwatch.md +++ b/docs/sources/features/datasources/cloudwatch.md @@ -78,11 +78,14 @@ CloudWatch Datasource Plugin provides the following queries you can specify in t edit view. They allow you to fill a variable's options list with things like `region`, `namespaces`, `metric names` and `dimension keys/values`. +In place of `region` you can specify `default` to use the default region configured in the datasource for the query, +e.g. `metrics(AWS/DynamoDB, default)` or `dimension_values(default, ..., ..., ...)`. + Name | Description ------- | -------- *regions()* | Returns a list of regions AWS provides their service. *namespaces()* | Returns a list of namespaces CloudWatch support. -*metrics(namespace, [region])* | Returns a list of metrics in the namespace. (specify region for custom metrics) +*metrics(namespace, [region])* | Returns a list of metrics in the namespace. (specify region or use "default" for custom metrics) *dimension_keys(namespace)* | Returns a list of dimension keys in the namespace. *dimension_values(region, namespace, metric, dimension_key)* | Returns a list of dimension values matching the specified `region`, `namespace`, `metric` and `dimension_key`. *ebs_volume_ids(region, instance_id)* | Returns a list of volume ids matching the specified `region`, `instance_id`. diff --git a/pkg/tsdb/cloudwatch/credentials.go b/pkg/tsdb/cloudwatch/credentials.go index 784f3b729ac..0c142bd4ea0 100644 --- a/pkg/tsdb/cloudwatch/credentials.go +++ b/pkg/tsdb/cloudwatch/credentials.go @@ -141,6 +141,11 @@ func ec2RoleProvider(sess *session.Session) credentials.Provider { } func (e *CloudWatchExecutor) getDsInfo(region string) *DatasourceInfo { + defaultRegion := e.DataSource.JsonData.Get("defaultRegion").MustString() + if region == "default" { + region = defaultRegion + } + authType := e.DataSource.JsonData.Get("authType").MustString() assumeRoleArn := e.DataSource.JsonData.Get("assumeRoleArn").MustString() accessKey := "" diff --git a/public/app/plugins/datasource/cloudwatch/datasource.js b/public/app/plugins/datasource/cloudwatch/datasource.js index ac4573ef43a..b0d37fd2186 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.js +++ b/public/app/plugins/datasource/cloudwatch/datasource.js @@ -39,7 +39,7 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { !!item.metricName && !_.isEmpty(item.statistics); }).map(function (item) { - item.region = templateSrv.replace(item.region, options.scopedVars); + item.region = templateSrv.replace(self.getActualRegion(item.region), options.scopedVars); item.namespace = templateSrv.replace(item.namespace, options.scopedVars); item.metricName = templateSrv.replace(item.metricName, options.scopedVars); item.dimensions = self.convertDimensionFormat(item.dimensions, options.scopeVars); @@ -165,21 +165,21 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { this.getMetrics = function (namespace, region) { return this.doMetricQueryRequest('metrics', { - region: templateSrv.replace(region), + region: templateSrv.replace(this.getActualRegion(region)), namespace: templateSrv.replace(namespace) }); }; this.getDimensionKeys = function(namespace, region) { return this.doMetricQueryRequest('dimension_keys', { - region: templateSrv.replace(region), + region: templateSrv.replace(this.getActualRegion(region)), namespace: templateSrv.replace(namespace) }); }; this.getDimensionValues = function(region, namespace, metricName, dimensionKey, filterDimensions) { return this.doMetricQueryRequest('dimension_values', { - region: templateSrv.replace(region), + region: templateSrv.replace(this.getActualRegion(region)), namespace: templateSrv.replace(namespace), metricName: templateSrv.replace(metricName), dimensionKey: templateSrv.replace(dimensionKey), @@ -189,14 +189,14 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { this.getEbsVolumeIds = function(region, instanceId) { return this.doMetricQueryRequest('ebs_volume_ids', { - region: templateSrv.replace(region), + region: templateSrv.replace(this.getActualRegion(region)), instanceId: templateSrv.replace(instanceId) }); }; this.getEc2InstanceAttribute = function(region, attributeName, filters) { return this.doMetricQueryRequest('ec2_instance_attribute', { - region: templateSrv.replace(region), + region: templateSrv.replace(this.getActualRegion(region)), attributeName: templateSrv.replace(attributeName), filters: filters }); @@ -267,7 +267,7 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { period = parseInt(period, 10); var parameters = { prefixMatching: annotation.prefixMatching, - region: templateSrv.replace(annotation.region), + region: templateSrv.replace(this.getActualRegion(annotation.region)), namespace: templateSrv.replace(annotation.namespace), metricName: templateSrv.replace(annotation.metricName), dimensions: this.convertDimensionFormat(annotation.dimensions, {}), @@ -341,6 +341,13 @@ function (angular, _, moment, dateMath, kbn, templatingVariable) { return this.defaultRegion; }; + this.getActualRegion = function(region) { + if (region === 'default' || _.isEmpty(region)) { + return this.getDefaultRegion(); + } + return region; + }; + this.getExpandedVariables = function(target, dimensionKey, variable, templateSrv) { /* if the all checkbox is marked we should add all values to the targets */ var allSelected = _.find(variable.options, {'selected': true, 'text': 'All'}); diff --git a/public/app/plugins/datasource/cloudwatch/query_parameter_ctrl.ts b/public/app/plugins/datasource/cloudwatch/query_parameter_ctrl.ts index 57a5fba443b..6bf22b0f2e7 100644 --- a/public/app/plugins/datasource/cloudwatch/query_parameter_ctrl.ts +++ b/public/app/plugins/datasource/cloudwatch/query_parameter_ctrl.ts @@ -28,7 +28,7 @@ export class CloudWatchQueryParameterCtrl { target.statistics = target.statistics || ['Average']; target.dimensions = target.dimensions || {}; target.period = target.period || ''; - target.region = target.region || ''; + target.region = target.region || 'default'; $scope.regionSegment = uiSegmentSrv.getSegmentForValue($scope.target.region, 'select region'); $scope.namespaceSegment = uiSegmentSrv.getSegmentForValue($scope.target.namespace, 'select namespace'); @@ -51,7 +51,7 @@ export class CloudWatchQueryParameterCtrl { $scope.removeStatSegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove stat --'}); if (_.isEmpty($scope.target.region)) { - $scope.target.region = $scope.datasource.getDefaultRegion(); + $scope.target.region = 'default'; } if (!$scope.onChange) { @@ -148,6 +148,10 @@ export class CloudWatchQueryParameterCtrl { $scope.getRegions = function() { return $scope.datasource.metricFindQuery('regions()') + .then(function(results) { + results.unshift({ text: 'default'}); + return results; + }) .then($scope.transformToSegments(true)); }; diff --git a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts index f278ce9305d..5eda60d5b9e 100644 --- a/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/cloudwatch/specs/datasource_specs.ts @@ -165,6 +165,55 @@ describe('CloudWatchDatasource', function() { }); }); + describe('When query region is "default"', function () { + it('should return the datasource region if empty or "default"', function() { + var defaultRegion = instanceSettings.jsonData.defaultRegion; + + expect(ctx.ds.getActualRegion()).to.be(defaultRegion); + expect(ctx.ds.getActualRegion('')).to.be(defaultRegion); + expect(ctx.ds.getActualRegion("default")).to.be(defaultRegion); + }); + + it('should return the specified region if specified', function() { + expect(ctx.ds.getActualRegion('some-fake-region-1')).to.be('some-fake-region-1'); + }); + + var requestParams; + beforeEach(function() { + ctx.ds.performTimeSeriesQuery = function(request) { + requestParams = request; + return ctx.$q.when({data: {}}); + }; + }); + + it('should query for the datasource region if empty or "default"', function(done) { + var query = { + range: { from: 'now-1h', to: 'now' }, + rangeRaw: { from: 1483228800, to: 1483232400 }, + targets: [ + { + region: 'default', + namespace: 'AWS/EC2', + metricName: 'CPUUtilization', + dimensions: { + InstanceId: 'i-12345678' + }, + statistics: ['Average'], + period: 300 + } + ] + }; + + ctx.ds.query(query).then(function(result) { + expect(requestParams.queries[0].region).to.be(instanceSettings.jsonData.defaultRegion); + done(); + }); + ctx.$rootScope.$apply(); + }); + + + }); + describe('When performing CloudWatch query for extended statistics', function() { var requestParams; @@ -348,6 +397,26 @@ describe('CloudWatchDatasource', function() { }); }); + describeMetricFindQuery('dimension_values(default,AWS/EC2,CPUUtilization,InstanceId)', scenario => { + scenario.setup(() => { + scenario.requestResponse = { + results: { + metricFindQuery: { + tables: [ + { rows: [['i-12345678', 'i-12345678']] } + ] + } + } + }; + }); + + it('should call __ListMetrics and return result', () => { + expect(scenario.result[0].text).to.contain('i-12345678'); + expect(scenario.request.queries[0].type).to.be('metricFindQuery'); + expect(scenario.request.queries[0].subtype).to.be('dimension_values'); + }); + }); + it('should caclculate the correct period', function () { var hourSec = 60 * 60; var daySec = hourSec * 24; From 66bc1fea2da651c28b288ebf7e062f0c68b26154 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 11 Dec 2017 09:45:17 +0100 Subject: [PATCH 42/44] changelog: adds note about closing #10131 --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8b1dc8bdce..6ba7b4a1aff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,9 @@ From `/etc/grafana/datasources` to `/etc/grafana/provisioning/datasources` when * **Dashboard**: Make it possible to start dashboards from search and dashboard list panel [#1871](https://github.com/grafana/grafana/issues/1871) * **Annotations**: Posting annotations now return the id of the annotation [#9798](https://github.com/grafana/grafana/issues/9798) * **Systemd**: Use systemd notification ready flag [#10024](https://github.com/grafana/grafana/issues/10024), thx [@jgrassler](https://github.com/jgrassler) -* **Github**: Use organizations_url provided from github to verify user belongs in org. [#10111](https://github.com/grafana/grafana/issues/10111), thx [@adiletmaratov](https://github.com/adiletmaratov) +* **Github**: Use organizations_url provided from github to verify user belongs in org. [#10111](https://github.com/grafana/grafana/issues/10111), thx +[@adiletmaratov](https://github.com/adiletmaratov) +* **Backend**: Fixed bug where Grafana exited before all sub routines where finished [#10131](https://github.com/grafana/grafana/issues/10131) ## Tech * **RabbitMq**: Remove support for publishing events to RabbitMQ [#9645](https://github.com/grafana/grafana/issues/9645) From c68e7c72835a0214bf49ecd93ef25c043ec861a4 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 7 Dec 2017 17:59:39 +0100 Subject: [PATCH 43/44] avatar: avoid concurrent map writes --- pkg/api/avatar/avatar.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/api/avatar/avatar.go b/pkg/api/avatar/avatar.go index fdf93d06b5d..6824e330f00 100644 --- a/pkg/api/avatar/avatar.go +++ b/pkg/api/avatar/avatar.go @@ -25,6 +25,8 @@ import ( "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/setting" "gopkg.in/macaron.v1" + + gocache "github.com/patrickmn/go-cache" ) var gravatarSource string @@ -92,7 +94,7 @@ func (this *Avatar) Update() (err error) { type CacheServer struct { notFound *Avatar - cache map[string]*Avatar + cache *gocache.Cache } func (this *CacheServer) mustInt(r *http.Request, defaultValue int, keys ...string) (v int) { @@ -110,7 +112,9 @@ func (this *CacheServer) Handler(ctx *macaron.Context) { var avatar *Avatar - if avatar, _ = this.cache[hash]; avatar == nil { + if obj, exist := this.cache.Get(hash); exist { + avatar = obj.(*Avatar) + } else { avatar = New(hash) } @@ -124,7 +128,7 @@ func (this *CacheServer) Handler(ctx *macaron.Context) { if avatar.notFound { avatar = this.notFound } else { - this.cache[hash] = avatar + this.cache.Add(hash, avatar, gocache.DefaultExpiration) } ctx.Resp.Header().Add("Content-Type", "image/jpeg") @@ -146,7 +150,7 @@ func NewCacheServer() *CacheServer { return &CacheServer{ notFound: newNotFound(), - cache: make(map[string]*Avatar), + cache: gocache.New(time.Hour, time.Hour*2), } } From 871b98c06b4b14f3a80e58fe36adffc9f5602b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 12 Dec 2017 12:56:40 +0100 Subject: [PATCH 44/44] graphite: minor fix for PR #10142 the query was being sent for every segmen t you selected before you completed the metric path --- public/app/plugins/datasource/graphite/graphite_query.ts | 4 ++++ public/app/plugins/datasource/graphite/query_ctrl.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/graphite/graphite_query.ts b/public/app/plugins/datasource/graphite/graphite_query.ts index 54504219dad..824b49f8860 100644 --- a/public/app/plugins/datasource/graphite/graphite_query.ts +++ b/public/app/plugins/datasource/graphite/graphite_query.ts @@ -120,6 +120,10 @@ export default class GraphiteQuery { this.segments.push({value: "select metric"}); } + hasSelectMetric() { + return this.segments[this.segments.length - 1].value === 'select metric'; + } + addFunction(newFunc) { this.functions.push(newFunc); this.moveAliasFuncLast(); diff --git a/public/app/plugins/datasource/graphite/query_ctrl.ts b/public/app/plugins/datasource/graphite/query_ctrl.ts index d00ac90fe65..a2e54903bcb 100644 --- a/public/app/plugins/datasource/graphite/query_ctrl.ts +++ b/public/app/plugins/datasource/graphite/query_ctrl.ts @@ -218,7 +218,7 @@ export class GraphiteQueryCtrl extends QueryCtrl { var oldTarget = this.queryModel.target.target; this.updateModelTarget(); - if (this.queryModel.target !== oldTarget) { + if (this.queryModel.target !== oldTarget && !this.queryModel.hasSelectMetric()) { this.panelCtrl.refresh(); } }