provisioing: add lookup table provisioned dashboards

This commit is contained in:
bergquist
2018-01-23 12:28:56 +01:00
parent 083abff297
commit 67a9e6a71d
15 changed files with 400 additions and 162 deletions

View File

@@ -3,4 +3,4 @@
# folder: ''
# type: file
# options:
# folder: /var/lib/grafana/dashboards
# folder: /var/lib/grafana/dashboards

View File

@@ -182,7 +182,7 @@ func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) Response {
}
}
dashItem := &dashboards.SaveDashboardItem{
dashItem := &dashboards.SaveDashboardDTO{
Dashboard: dash,
Message: cmd.Message,
OrgId: c.OrgId,

View File

@@ -21,15 +21,25 @@ import (
)
type fakeDashboardRepo struct {
inserted []*dashboards.SaveDashboardItem
inserted []*dashboards.SaveDashboardDTO
provisioned []*m.DashboardProvisioning
getDashboard []*m.Dashboard
}
func (repo *fakeDashboardRepo) SaveDashboard(json *dashboards.SaveDashboardItem) (*m.Dashboard, error) {
func (repo *fakeDashboardRepo) SaveDashboard(json *dashboards.SaveDashboardDTO) (*m.Dashboard, error) {
repo.inserted = append(repo.inserted, json)
return json.Dashboard, nil
}
func (repo *fakeDashboardRepo) SaveProvisionedDashboard(dto *dashboards.SaveDashboardDTO, provisioning *m.DashboardProvisioning) (*m.Dashboard, error) {
repo.inserted = append(repo.inserted, dto)
return dto.Dashboard, nil
}
func (repo *fakeDashboardRepo) GetProvisionedDashboardData(name string) ([]*m.DashboardProvisioning, error) {
return repo.provisioned, nil
}
var fakeRepo *fakeDashboardRepo
func TestDashboardApiEndpoint(t *testing.T) {

View File

@@ -0,0 +1,2 @@
package models

View File

@@ -167,6 +167,21 @@ type SaveDashboardCommand struct {
Result *Dashboard
}
type DashboardProvisioning struct {
Id int64
DashboardId int64
Name string
ExternalId string
Updated time.Time
}
type SaveProvisionedDashboardCommand struct {
DashboardCmd *SaveDashboardCommand
DashboardProvisioning *DashboardProvisioning
Result *Dashboard
}
type DeleteDashboardCommand struct {
Id int64
OrgId int64
@@ -209,3 +224,9 @@ type GetDashboardSlugByIdQuery struct {
Id int64
Result string
}
type GetProvisionedDashboardDataQuery struct {
Name string
Result []*DashboardProvisioning
}

View File

@@ -9,7 +9,9 @@ import (
)
type Repository interface {
SaveDashboard(*SaveDashboardItem) (*models.Dashboard, error)
SaveDashboard(*SaveDashboardDTO) (*models.Dashboard, error)
SaveProvisionedDashboard(dto *SaveDashboardDTO, provisioning *models.DashboardProvisioning) (*models.Dashboard, error)
GetProvisionedDashboardData(name string) ([]*models.DashboardProvisioning, error)
}
var repositoryInstance Repository
@@ -22,7 +24,7 @@ func SetRepository(rep Repository) {
repositoryInstance = rep
}
type SaveDashboardItem struct {
type SaveDashboardDTO struct {
OrgId int64
UpdatedAt time.Time
UserId int64
@@ -33,15 +35,25 @@ type SaveDashboardItem struct {
type DashboardRepository struct{}
func (dr *DashboardRepository) SaveDashboard(json *SaveDashboardItem) (*models.Dashboard, error) {
dashboard := json.Dashboard
func (dr *DashboardRepository) GetProvisionedDashboardData(name string) ([]*models.DashboardProvisioning, error) {
cmd := &models.GetProvisionedDashboardDataQuery{Name: name}
err := bus.Dispatch(cmd)
if err != nil {
return nil, err
}
return cmd.Result, nil
}
func (dr *DashboardRepository) buildSaveDashboardCommand(dto *SaveDashboardDTO) (*models.SaveDashboardCommand, error) {
dashboard := dto.Dashboard
if dashboard.Title == "" {
return nil, models.ErrDashboardTitleEmpty
}
validateAlertsCmd := alerting.ValidateDashboardAlertsCommand{
OrgId: json.OrgId,
OrgId: dto.OrgId,
Dashboard: dashboard,
}
@@ -49,33 +61,77 @@ func (dr *DashboardRepository) SaveDashboard(json *SaveDashboardItem) (*models.D
return nil, models.ErrDashboardContainsInvalidAlertData
}
cmd := models.SaveDashboardCommand{
cmd := &models.SaveDashboardCommand{
Dashboard: dashboard.Data,
Message: json.Message,
OrgId: json.OrgId,
Overwrite: json.Overwrite,
UserId: json.UserId,
Message: dto.Message,
OrgId: dto.OrgId,
Overwrite: dto.Overwrite,
UserId: dto.UserId,
FolderId: dashboard.FolderId,
IsFolder: dashboard.IsFolder,
}
if !json.UpdatedAt.IsZero() {
cmd.UpdatedAt = json.UpdatedAt
if !dto.UpdatedAt.IsZero() {
cmd.UpdatedAt = dto.UpdatedAt
}
err := bus.Dispatch(&cmd)
if err != nil {
return nil, err
}
return cmd, nil
}
func (dr *DashboardRepository) updateAlerting(cmd *models.SaveDashboardCommand, dto *SaveDashboardDTO) error {
alertCmd := alerting.UpdateDashboardAlertsCommand{
OrgId: json.OrgId,
UserId: json.UserId,
OrgId: dto.OrgId,
UserId: dto.UserId,
Dashboard: cmd.Result,
}
if err := bus.Dispatch(&alertCmd); err != nil {
return nil, models.ErrDashboardFailedToUpdateAlertData
return models.ErrDashboardFailedToUpdateAlertData
}
return nil
}
func (dr *DashboardRepository) SaveProvisionedDashboard(dto *SaveDashboardDTO, provisioning *models.DashboardProvisioning) (*models.Dashboard, error) {
cmd, err := dr.buildSaveDashboardCommand(dto)
if err != nil {
return nil, err
}
saveCmd := &models.SaveProvisionedDashboardCommand{
DashboardCmd: cmd,
DashboardProvisioning: provisioning,
}
// dashboard
err = bus.Dispatch(saveCmd)
if err != nil {
return nil, err
}
//alerts
err = dr.updateAlerting(cmd, dto)
if err != nil {
return nil, err
}
return cmd.Result, nil
}
func (dr *DashboardRepository) SaveDashboard(dto *SaveDashboardDTO) (*models.Dashboard, error) {
cmd, err := dr.buildSaveDashboardCommand(dto)
if err != nil {
return nil, err
}
err = bus.Dispatch(&cmd)
if err != nil {
return nil, err
}
err = dr.updateAlerting(cmd, dto)
if err != nil {
return nil, err
}
return cmd.Result, nil

View File

@@ -14,17 +14,17 @@ func NewDashboardCache() *dashboardCache {
return &dashboardCache{internalCache: gocache.New(5*time.Minute, 30*time.Minute)}
}
func (fr *dashboardCache) addDashboardCache(key string, json *dashboards.SaveDashboardItem) {
func (fr *dashboardCache) addDashboardCache(key string, json *dashboards.SaveDashboardDTO) {
fr.internalCache.Add(key, json, time.Minute*10)
}
func (fr *dashboardCache) getCache(key string) (*dashboards.SaveDashboardItem, bool) {
func (fr *dashboardCache) getCache(key string) (*dashboards.SaveDashboardDTO, bool) {
obj, exist := fr.internalCache.Get(key)
if !exist {
return nil, exist
}
dash, ok := obj.(*dashboards.SaveDashboardItem)
dash, ok := obj.(*dashboards.SaveDashboardDTO)
if !ok {
return nil, ok
}

View File

@@ -25,12 +25,12 @@ var (
)
type fileReader struct {
Cfg *DashboardsAsConfig
Path string
log log.Logger
dashboardRepo dashboards.Repository
cache *dashboardCache
createWalk func(fr *fileReader, folderId int64) filepath.WalkFunc
Cfg *DashboardsAsConfig
Path string
log log.Logger
dashboardRepo dashboards.Repository
cache *dashboardCache
createWalk func(fr *fileReader, folderId int64) filepath.WalkFunc
}
func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) {
@@ -50,28 +50,28 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade
}
return &fileReader{
Cfg: cfg,
Path: path,
log: log,
dashboardRepo: dashboards.GetRepository(),
cache: NewDashboardCache(),
createWalk: createWalkFn,
Cfg: cfg,
Path: path,
log: log,
dashboardRepo: dashboards.GetRepository(),
cache: NewDashboardCache(),
createWalk: createWalkFn,
}, nil
}
func (fr *fileReader) ReadAndListen(ctx context.Context) error {
ticker := time.NewTicker(checkDiskForChangesInterval)
if err := fr.startWalkingDisk(); err != nil {
fr.log.Error("failed to search for dashboards", "error", err)
}
ticker := time.NewTicker(checkDiskForChangesInterval)
running := false
for {
select {
case <-ticker.C:
if !running { // avoid walking the filesystem in parallel. incase fs is very slow.
if !running { // avoid walking the filesystem in parallel. in-case fs is very slow.
running = true
go func() {
if err := fr.startWalkingDisk(); err != nil {
@@ -115,7 +115,7 @@ func getOrCreateFolderId(cfg *DashboardsAsConfig, repo dashboards.Repository) (i
// dashboard folder not found. create one.
if err == models.ErrDashboardNotFound {
dash := &dashboards.SaveDashboardItem{}
dash := &dashboards.SaveDashboardDTO{}
dash.Dashboard = models.NewDashboard(cfg.Folder)
dash.Dashboard.IsFolder = true
dash.Overwrite = true
@@ -129,7 +129,7 @@ func getOrCreateFolderId(cfg *DashboardsAsConfig, repo dashboards.Repository) (i
}
if !cmd.Result.IsFolder {
return 0, fmt.Errorf("Got invalid response. Expected folder, found dashboard")
return 0, fmt.Errorf("got invalid response. expected folder, found dashboard")
}
return cmd.Result.Id, nil
@@ -188,7 +188,7 @@ func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc {
// if we don't have the dashboard in the db, save it!
if err == models.ErrDashboardNotFound {
fr.log.Debug("saving new dashboard", "file", path)
_, err = fr.dashboardRepo.SaveDashboard(dash)
err = saveDashboard(fr, path, dash, fileInfo.ModTime())
return err
}
@@ -203,11 +203,18 @@ func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc {
}
fr.log.Debug("loading dashboard from disk into database.", "file", path)
_, err = fr.dashboardRepo.SaveDashboard(dash)
err = saveDashboard(fr, path, dash, fileInfo.ModTime())
return err
}
}
func saveDashboard(fr *fileReader, path string, dash *dashboards.SaveDashboardDTO, modTime time.Time) error {
//dash.Extras["provisioning.filepath"] = path
_, err := fr.dashboardRepo.SaveDashboard(dash)
return err
}
func validateWalkablePath(fileInfo os.FileInfo) (bool, error) {
if fileInfo.IsDir() {
@@ -224,7 +231,7 @@ func validateWalkablePath(fileInfo os.FileInfo) (bool, error) {
return true, nil
}
func (fr *fileReader) readDashboardFromFile(path string, folderId int64) (*dashboards.SaveDashboardItem, error) {
func (fr *fileReader) readDashboardFromFile(path string, folderId int64) (*dashboards.SaveDashboardDTO, error) {
reader, err := os.Open(path)
if err != nil {
return nil, err

View File

@@ -241,15 +241,25 @@ func (ffi FakeFileInfo) Sys() interface{} {
}
type fakeDashboardRepo struct {
inserted []*dashboards.SaveDashboardItem
inserted []*dashboards.SaveDashboardDTO
provisioned []*models.DashboardProvisioning
getDashboard []*models.Dashboard
}
func (repo *fakeDashboardRepo) SaveDashboard(json *dashboards.SaveDashboardItem) (*models.Dashboard, error) {
func (repo *fakeDashboardRepo) SaveDashboard(json *dashboards.SaveDashboardDTO) (*models.Dashboard, error) {
repo.inserted = append(repo.inserted, json)
return json.Dashboard, nil
}
func (repo *fakeDashboardRepo) GetProvisionedDashboardData(name string) ([]*models.DashboardProvisioning, error) {
return repo.provisioned, nil
}
func (repo *fakeDashboardRepo) SaveProvisionedDashboard(dto *dashboards.SaveDashboardDTO, provisioning *models.DashboardProvisioning) (*models.Dashboard, error) {
repo.inserted = append(repo.inserted, dto)
return dto.Dashboard, nil
}
func mockGetDashboardQuery(cmd *models.GetDashboardQuery) error {
for _, d := range fakeRepo.getDashboard {
if d.Slug == cmd.Slug {

View File

@@ -18,8 +18,8 @@ type DashboardsAsConfig struct {
Options map[string]interface{} `json:"options" yaml:"options"`
}
func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig, folderId int64) (*dashboards.SaveDashboardItem, error) {
dash := &dashboards.SaveDashboardItem{}
func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig, folderId int64) (*dashboards.SaveDashboardDTO, error) {
dash := &dashboards.SaveDashboardDTO{}
dash.Dashboard = models.NewDashboardFromJson(data)
dash.UpdatedAt = lastModified
dash.Overwrite = true

View File

@@ -22,121 +22,126 @@ func init() {
func SaveDashboard(cmd *m.SaveDashboardCommand) error {
return inTransaction(func(sess *DBSession) error {
dash := cmd.GetDashboardModel()
return saveDashboard(sess, cmd)
})
}
// try get existing dashboard
var existing, sameTitle m.Dashboard
func saveDashboard(sess *DBSession, cmd *m.SaveDashboardCommand) error {
dash := cmd.GetDashboardModel()
if dash.Id > 0 {
dashWithIdExists, err := sess.Where("id=? AND org_id=?", dash.Id, dash.OrgId).Get(&existing)
if err != nil {
// try get existing dashboard
var existing, sameTitle m.Dashboard
if dash.Id > 0 {
dashWithIdExists, err := sess.Where("id=? AND org_id=?", dash.Id, dash.OrgId).Get(&existing)
if err != nil {
return err
}
if !dashWithIdExists {
return m.ErrDashboardNotFound
}
// check for is someone else has written in between
if dash.Version != existing.Version {
if cmd.Overwrite {
dash.Version = existing.Version
} else {
return m.ErrDashboardVersionMismatch
}
}
// do not allow plugin dashboard updates without overwrite flag
if existing.PluginId != "" && cmd.Overwrite == false {
return m.UpdatePluginDashboardError{PluginId: existing.PluginId}
}
}
sameTitleExists, err := sess.Where("org_id=? AND slug=?", dash.OrgId, dash.Slug).Get(&sameTitle)
if err != nil {
return err
}
if sameTitleExists {
// another dashboard with same name
if dash.Id != sameTitle.Id {
if cmd.Overwrite {
dash.Id = sameTitle.Id
dash.Version = sameTitle.Version
} else {
return m.ErrDashboardWithSameNameExists
}
}
}
err = setHasAcl(sess, dash)
if err != nil {
return err
}
parentVersion := dash.Version
affectedRows := int64(0)
if dash.Id == 0 {
dash.Version = 1
metrics.M_Api_Dashboard_Insert.Inc()
dash.Data.Set("version", dash.Version)
affectedRows, err = sess.Insert(dash)
} else {
dash.Version++
dash.Data.Set("version", dash.Version)
if !cmd.UpdatedAt.IsZero() {
dash.Updated = cmd.UpdatedAt
}
affectedRows, err = sess.MustCols("folder_id", "has_acl").Id(dash.Id).Update(dash)
}
if err != nil {
return err
}
if affectedRows == 0 {
return m.ErrDashboardNotFound
}
dashVersion := &m.DashboardVersion{
DashboardId: dash.Id,
ParentVersion: parentVersion,
RestoredFrom: cmd.RestoredFrom,
Version: dash.Version,
Created: time.Now(),
CreatedBy: dash.UpdatedBy,
Message: cmd.Message,
Data: dash.Data,
}
// insert version entry
if affectedRows, err = sess.Insert(dashVersion); err != nil {
return err
} else if affectedRows == 0 {
return m.ErrDashboardNotFound
}
// delete existing tags
_, err = sess.Exec("DELETE FROM dashboard_tag WHERE dashboard_id=?", dash.Id)
if err != nil {
return err
}
// insert new tags
tags := dash.GetTags()
if len(tags) > 0 {
for _, tag := range tags {
if _, err := sess.Insert(&DashboardTag{DashboardId: dash.Id, Term: tag}); err != nil {
return err
}
if !dashWithIdExists {
return m.ErrDashboardNotFound
}
// check for is someone else has written in between
if dash.Version != existing.Version {
if cmd.Overwrite {
dash.Version = existing.Version
} else {
return m.ErrDashboardVersionMismatch
}
}
// do not allow plugin dashboard updates without overwrite flag
if existing.PluginId != "" && cmd.Overwrite == false {
return m.UpdatePluginDashboardError{PluginId: existing.PluginId}
}
}
}
sameTitleExists, err := sess.Where("org_id=? AND slug=?", dash.OrgId, dash.Slug).Get(&sameTitle)
if err != nil {
return err
}
cmd.Result = dash
if sameTitleExists {
// another dashboard with same name
if dash.Id != sameTitle.Id {
if cmd.Overwrite {
dash.Id = sameTitle.Id
dash.Version = sameTitle.Version
} else {
return m.ErrDashboardWithSameNameExists
}
}
}
err = setHasAcl(sess, dash)
if err != nil {
return err
}
parentVersion := dash.Version
affectedRows := int64(0)
if dash.Id == 0 {
dash.Version = 1
metrics.M_Api_Dashboard_Insert.Inc()
dash.Data.Set("version", dash.Version)
affectedRows, err = sess.Insert(dash)
} else {
dash.Version++
dash.Data.Set("version", dash.Version)
if !cmd.UpdatedAt.IsZero() {
dash.Updated = cmd.UpdatedAt
}
affectedRows, err = sess.MustCols("folder_id", "has_acl").Id(dash.Id).Update(dash)
}
if err != nil {
return err
}
if affectedRows == 0 {
return m.ErrDashboardNotFound
}
dashVersion := &m.DashboardVersion{
DashboardId: dash.Id,
ParentVersion: parentVersion,
RestoredFrom: cmd.RestoredFrom,
Version: dash.Version,
Created: time.Now(),
CreatedBy: dash.UpdatedBy,
Message: cmd.Message,
Data: dash.Data,
}
// insert version entry
if affectedRows, err = sess.Insert(dashVersion); err != nil {
return err
} else if affectedRows == 0 {
return m.ErrDashboardNotFound
}
// delete existing tags
_, err = sess.Exec("DELETE FROM dashboard_tag WHERE dashboard_id=?", dash.Id)
if err != nil {
return err
}
// insert new tags
tags := dash.GetTags()
if len(tags) > 0 {
for _, tag := range tags {
if _, err := sess.Insert(&DashboardTag{DashboardId: dash.Id, Term: tag}); err != nil {
return err
}
}
}
cmd.Result = dash
return err
})
return err
}
func setHasAcl(sess *DBSession, dash *m.Dashboard) error {

View File

@@ -0,0 +1,66 @@
package sqlstore
import (
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/models"
)
func init() {
bus.AddHandler("sql", GetProvisionedDashboardDataQuery)
bus.AddHandler("sql", SaveProvisionedDashboard)
}
type DashboardExtras struct {
Id int64
DashboardId int64
Key string
Value string
}
func SaveProvisionedDashboard(cmd *models.SaveProvisionedDashboardCommand) error {
return inTransaction(func(sess *DBSession) error {
err := saveDashboard(sess, cmd.DashboardCmd)
if err != nil {
return err
}
cmd.Result = cmd.DashboardCmd.Result
return saveProvionedData(sess, cmd.DashboardProvisioning)
})
}
func saveProvionedData(sess *DBSession, cmd *models.DashboardProvisioning) error {
results := &models.DashboardProvisioning{}
exist, err := sess.Where("dashboard_id=?", cmd.DashboardId).Get(results)
if err != nil {
return err
}
cmd.Id = results.Id
cmd.Updated = time.Now()
println("exists", exist)
if exist {
_, err = sess.ID(results.Id).Update(cmd)
} else {
_, err = sess.Insert(cmd)
}
return err
}
func GetProvisionedDashboardDataQuery(cmd *models.GetProvisionedDashboardDataQuery) error {
var result []*models.DashboardProvisioning
if err := x.Where("name = ?", cmd.Name).Find(&result); err != nil {
return err
}
cmd.Result = result
return nil
}

View File

@@ -0,0 +1,48 @@
package sqlstore
import (
"testing"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
func TestDashboardProvisioningTest(t *testing.T) {
Convey("Testing Dashboard provisioning", t, func() {
InitTestDB(t)
saveDashboardCmd := &models.SaveDashboardCommand{
OrgId: 1,
FolderId: 0,
IsFolder: false,
Dashboard: simplejson.NewFromAny(map[string]interface{}{
"id": nil,
"title": "test dashboard",
}),
}
Convey("Saving dashboards with extras", func() {
cmd := &models.SaveProvisionedDashboardCommand{
DashboardCmd: saveDashboardCmd,
DashboardProvisioning: &models.DashboardProvisioning{
Name: "default",
ExternalId: "/var/grafana.json",
},
}
err := SaveProvisionedDashboard(cmd)
So(err, ShouldBeNil)
So(cmd.Result, ShouldNotBeNil)
So(cmd.Result.Id, ShouldNotEqual, 0)
Convey("Can query for provisioned dashboards", func() {
query := &models.GetProvisionedDashboardDataQuery{Name: "default"}
err := GetProvisionedDashboardDataQuery(query)
So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 1)
})
})
})
}

View File

@@ -150,4 +150,21 @@ func addDashboardMigration(mg *Migrator) {
mg.AddMigration("Add column has_acl in dashboard", NewAddColumnMigration(dashboardV2, &Column{
Name: "has_acl", Type: DB_Bool, Nullable: false, Default: "0",
}))
dashboardExtrasTable := Table{
Name: "dashboard_provisioning",
Columns: []*Column{
{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
{Name: "dashboard_id", Type: DB_BigInt, Nullable: true},
{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
{Name: "external_id", Type: DB_Text, Nullable: false},
{Name: "updated", Type: DB_Int, Nullable: false},
},
Indices: []*Index{
{Cols: []string{"dashboard_id"}},
{Cols: []string{"dashboard_id", "name"}, Type: IndexType},
},
}
mg.AddMigration("create dashboard_provisioning", NewAddTableMigration(dashboardExtrasTable))
}

View File

@@ -1,8 +1,6 @@
package sqlstore
import (
"fmt"
"github.com/grafana/grafana/pkg/bus"
m "github.com/grafana/grafana/pkg/models"
)
@@ -27,8 +25,6 @@ func CreatePlaylist(cmd *m.CreatePlaylistCommand) error {
_, err = x.Insert(&playlist)
fmt.Printf("%v", playlist.Id)
playlistItems := make([]m.PlaylistItem, 0)
for _, item := range cmd.Items {
playlistItems = append(playlistItems, m.PlaylistItem{