datasources as cfg: tests for insert/updating datasources

conf/datasources.yaml

@@ -1,28 +1,30 @@
-# - name: Graphite202
+purgeOtherDatasources: false
-#   type: graphite
+datasources:
-#   access: proxy
+  - name: Graphite202
-#   url: http://localhost:8080
+    type: graphite
-#   password:
+    access: proxy
-#   user: 
+    url: http://localhost:8080
-#   database:
+    password:
-#   basicAuth:
+    user: 
-#   basicAuthUser:
+    database:
-#   basicAuthPassword:
+    basicAuth:
-#   withCredentials:
+    basicAuthUser:
-#   isDefault: true
+    basicAuthPassword:
-#   jsonData: {}
+    withCredentials:
-#   secureJsonFields: {}
+    isDefault: true
-# - name: Prometheus
+    jsonData: {}
-#   type: prometheus
+    secureJsonFields: {}
-#   access: proxy
+  - name: Prometheus
-#   url: http://localhost:9090
+    type: prometheus
-#   password:
+    access: proxy
-#   user: 
+    url: http://localhost:9090
-#   database:
+    password:
-#   basicAuth:
+    user: 
-#   basicAuthUser:
+    database:
-#   basicAuthPassword:
+    basicAuth:
-#   withCredentials:
+    basicAuthUser:
-#   isDefault: true
+    basicAuthPassword:
-#   jsonData: {}
+    withCredentials:
-#   secureJsonFields: {}
+    isDefault: true
+    jsonData: {}
+    secureJsonFields: {}

pkg/models/datasource.go

@@ -157,6 +157,10 @@ type GetDataSourcesQuery struct {
 	Result []*DataSource
 }
 
+type GetAllDataSourcesQuery struct {
+	Result []*DataSource
+}
+
 type GetDataSourceByIdQuery struct {
 	Id     int64
 	OrgId  int64

pkg/services/sqlstore/datasource.go

@@ -13,6 +13,7 @@ import (
 
 func init() {
 	bus.AddHandler("sql", GetDataSources)
+	bus.AddHandler("sql", GetAllDataSources)
 	bus.AddHandler("sql", AddDataSource)
 	bus.AddHandler("sql", DeleteDataSourceById)
 	bus.AddHandler("sql", DeleteDataSourceByName)
@@ -54,6 +55,13 @@ func GetDataSources(query *m.GetDataSourcesQuery) error {
 	return sess.Find(&query.Result)
 }
 
+func GetAllDataSources(query *m.GetAllDataSourcesQuery) error {
+	sess := x.Limit(1000, 0).Asc("name")
+
+	query.Result = make([]*m.DataSource, 0)
+	return sess.Find(&query.Result)
+}
+
 func DeleteDataSourceById(cmd *m.DeleteDataSourceByIdCommand) error {
 	return inTransaction(func(sess *DBSession) error {
 		var rawSql = "DELETE FROM data_source WHERE id=? and org_id=?"

pkg/setting/datasources/datasources.go

@@ -0,0 +1,182 @@
+package datasources
+
+import (
+	"io"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/log"
+
+	"github.com/grafana/grafana/pkg/models"
+	yaml "gopkg.in/yaml.v2"
+)
+
+// TODO: secure jsonData
+// TODO: auto reload on file changes
+
+type DatasourcesAsConfig struct {
+	PurgeOtherDatasources bool
+	Datasources           []models.DataSource
+}
+
+func Init(configPath string) (error, io.Closer) {
+
+	dc := NewDatasourceConfiguration()
+	dc.applyChanges(configPath)
+
+	return nil, ioutil.NopCloser(nil)
+}
+
+type DatasourceConfigurator struct {
+	log         log.Logger
+	cfgProvider configProvider
+	repository  datasourceRepository
+}
+
+func NewDatasourceConfiguration() DatasourceConfigurator {
+	return newDatasourceConfiguration(log.New("setting.datasource"), diskConfigReader{}, sqlDatasourceRepository{})
+}
+
+func newDatasourceConfiguration(log log.Logger, cfgProvider configProvider, repo datasourceRepository) DatasourceConfigurator {
+	return DatasourceConfigurator{
+		log:         log.New("setting.datasource"),
+		repository:  repo,
+		cfgProvider: cfgProvider,
+	}
+}
+
+func (dc *DatasourceConfigurator) applyChanges(configPath string) error {
+	datasources, err := dc.cfgProvider.readConfig(configPath)
+	if err != nil {
+		return err
+	}
+
+	//read all datasources
+	//delete datasources not in list
+
+	for _, ds := range datasources.Datasources {
+		if ds.OrgId == 0 {
+			ds.OrgId = 1
+		}
+
+		query := &models.GetDataSourceByNameQuery{Name: ds.Name, OrgId: ds.OrgId}
+		err := dc.repository.get(query)
+		if err != nil && err != models.ErrDataSourceNotFound {
+			return err
+		}
+
+		if query.Result == nil {
+			dc.log.Info("inserting ", "name", ds.Name)
+			insertCmd := createInsertCommand(ds)
+			if err := dc.repository.insert(insertCmd); err != nil {
+				return err
+			}
+		} else {
+			dc.log.Info("updating", "name", ds.Name)
+			updateCmd := createUpdateCommand(ds, query.Result.Id)
+			if err := dc.repository.update(updateCmd); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func createInsertCommand(ds models.DataSource) *models.AddDataSourceCommand {
+	return &models.AddDataSourceCommand{
+		OrgId:             ds.OrgId,
+		Name:              ds.Name,
+		Type:              ds.Type,
+		Access:            ds.Access,
+		Url:               ds.Url,
+		Password:          ds.Password,
+		User:              ds.User,
+		Database:          ds.Database,
+		BasicAuth:         ds.BasicAuth,
+		BasicAuthUser:     ds.BasicAuthUser,
+		BasicAuthPassword: ds.BasicAuthPassword,
+		WithCredentials:   ds.WithCredentials,
+		IsDefault:         ds.IsDefault,
+		JsonData:          ds.JsonData,
+	}
+}
+
+func createUpdateCommand(ds models.DataSource, id int64) *models.UpdateDataSourceCommand {
+	return &models.UpdateDataSourceCommand{
+		Id:                id,
+		OrgId:             ds.OrgId,
+		Name:              ds.Name,
+		Type:              ds.Type,
+		Access:            ds.Access,
+		Url:               ds.Url,
+		Password:          ds.Password,
+		User:              ds.User,
+		Database:          ds.Database,
+		BasicAuth:         ds.BasicAuth,
+		BasicAuthUser:     ds.BasicAuthUser,
+		BasicAuthPassword: ds.BasicAuthPassword,
+		WithCredentials:   ds.WithCredentials,
+		IsDefault:         ds.IsDefault,
+		JsonData:          ds.JsonData,
+	}
+}
+
+type datasourceRepository interface {
+	insert(*models.AddDataSourceCommand) error
+	update(*models.UpdateDataSourceCommand) error
+	delete(*models.DeleteDataSourceByIdCommand) error
+	get(*models.GetDataSourceByNameQuery) error
+	loadAllDatasources() ([]*models.DataSource, error)
+}
+
+type configProvider interface {
+	readConfig(string) (*DatasourcesAsConfig, error)
+}
+
+type sqlDatasourceRepository struct{}
+type diskConfigReader struct{}
+
+func (diskConfigReader) readConfig(path string) (*DatasourcesAsConfig, error) {
+	filename, _ := filepath.Abs(path)
+	yamlFile, err := ioutil.ReadFile(filename)
+
+	if err != nil {
+		return nil, err
+	}
+
+	var datasources *DatasourcesAsConfig
+
+	err = yaml.Unmarshal(yamlFile, &datasources)
+	if err != nil {
+		return nil, err
+	}
+
+	return datasources, nil
+}
+
+func (sqlDatasourceRepository) delete(cmd *models.DeleteDataSourceByIdCommand) error {
+	return bus.Dispatch(cmd)
+}
+
+func (sqlDatasourceRepository) update(cmd *models.UpdateDataSourceCommand) error {
+	return bus.Dispatch(cmd)
+}
+
+func (sqlDatasourceRepository) insert(cmd *models.AddDataSourceCommand) error {
+	return bus.Dispatch(cmd)
+}
+
+func (sqlDatasourceRepository) get(cmd *models.GetDataSourceByNameQuery) error {
+	return bus.Dispatch(cmd)
+}
+
+func (sqlDatasourceRepository) loadAllDatasources() ([]*models.DataSource, error) {
+	dss := &models.GetAllDataSourcesQuery{}
+	if err := bus.Dispatch(dss); err != nil {
+		return nil, err
+	}
+
+	return dss.Result, nil
+}

pkg/setting/datasources/datasources_test.go

@@ -0,0 +1,152 @@
+package datasources
+
+import (
+	"testing"
+
+	"github.com/grafana/grafana/pkg/log"
+	"github.com/grafana/grafana/pkg/models"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var logger log.Logger = log.New("fake.logger")
+
+func TestDatasourceAsConfig(t *testing.T) {
+	Convey("Testing datasource as configuration", t, func() {
+		fakeCfg := &fakeConfig{}
+
+		fakeRepo := &fakeRepository{}
+
+		Convey("One configured datasource", func() {
+			fakeCfg.cfg = &DatasourcesAsConfig{
+				PurgeOtherDatasources: false,
+				Datasources: []models.DataSource{
+					models.DataSource{Name: "graphite", OrgId: 1},
+				},
+			}
+
+			Convey("no datasource in database", func() {
+				dc := newDatasourceConfiguration(logger, fakeCfg, fakeRepo)
+				err := dc.applyChanges("mock/config.yaml")
+				if err != nil {
+					t.Fatalf("applyChanges return an error %v", err)
+				}
+
+				So(len(fakeRepo.deleted), ShouldEqual, 0)
+				So(len(fakeRepo.inserted), ShouldEqual, 1)
+				So(len(fakeRepo.updated), ShouldEqual, 0)
+			})
+
+			Convey("One datasource in database with same name", func() {
+				fakeRepo.loadAll = []*models.DataSource{
+					&models.DataSource{Name: "graphite", OrgId: 1, Id: 1},
+				}
+
+				Convey("should update one datasource", func() {
+					dc := newDatasourceConfiguration(logger, fakeCfg, fakeRepo)
+					err := dc.applyChanges("mock/config.yaml")
+					if err != nil {
+						t.Fatalf("applyChanges return an error %v", err)
+					}
+
+					So(len(fakeRepo.deleted), ShouldEqual, 0)
+					So(len(fakeRepo.inserted), ShouldEqual, 0)
+					So(len(fakeRepo.updated), ShouldEqual, 1)
+				})
+			})
+
+			Convey("One datasource in database with new name", func() {
+				fakeRepo.loadAll = []*models.DataSource{
+					&models.DataSource{Name: "old-graphite", OrgId: 1, Id: 1},
+				}
+
+				Convey("should update one datasource", func() {
+					dc := newDatasourceConfiguration(logger, fakeCfg, fakeRepo)
+					err := dc.applyChanges("mock/config.yaml")
+					if err != nil {
+						t.Fatalf("applyChanges return an error %v", err)
+					}
+
+					So(len(fakeRepo.deleted), ShouldEqual, 0)
+					So(len(fakeRepo.inserted), ShouldEqual, 1)
+					So(len(fakeRepo.updated), ShouldEqual, 0)
+				})
+			})
+		})
+
+		Convey("Two configured datasource and purge others ", func() {
+			fakeCfg.cfg = &DatasourcesAsConfig{
+				PurgeOtherDatasources: true,
+				Datasources: []models.DataSource{
+					models.DataSource{Name: "graphite", OrgId: 1},
+					models.DataSource{Name: "prometheus", OrgId: 1},
+				},
+			}
+
+			Convey("two other datasources in database", func() {
+				fakeRepo.loadAll = []*models.DataSource{
+					&models.DataSource{Name: "old-graphite", OrgId: 1, Id: 1},
+					&models.DataSource{Name: "old-graphite2", OrgId: 1, Id: 2},
+				}
+
+				Convey("should have two new datasources", func() {
+					dc := newDatasourceConfiguration(logger, fakeCfg, fakeRepo)
+					err := dc.applyChanges("mock/config.yaml")
+					if err != nil {
+						t.Fatalf("applyChanges return an error %v", err)
+					}
+
+					So(len(fakeRepo.deleted), ShouldEqual, 2)
+					So(len(fakeRepo.inserted), ShouldEqual, 2)
+					So(len(fakeRepo.updated), ShouldEqual, 0)
+				})
+			})
+		})
+	})
+}
+
+type fakeRepository struct {
+	inserted []*models.AddDataSourceCommand
+	deleted  []*models.DeleteDataSourceByIdCommand
+	updated  []*models.UpdateDataSourceCommand
+
+	loadAll []*models.DataSource
+}
+
+type fakeConfig struct {
+	cfg *DatasourcesAsConfig
+}
+
+func (fc *fakeConfig) readConfig(path string) (*DatasourcesAsConfig, error) {
+	return fc.cfg, nil
+}
+
+func (fc *fakeRepository) delete(cmd *models.DeleteDataSourceByIdCommand) error {
+	fc.deleted = append(fc.deleted, cmd)
+	return nil
+}
+
+func (fc *fakeRepository) update(cmd *models.UpdateDataSourceCommand) error {
+	fc.updated = append(fc.updated, cmd)
+	return nil
+}
+
+func (fc *fakeRepository) insert(cmd *models.AddDataSourceCommand) error {
+	fc.inserted = append(fc.inserted, cmd)
+	return nil
+}
+
+func (fc *fakeRepository) loadAllDatasources() ([]*models.DataSource, error) {
+	return fc.loadAll, nil
+}
+
+func (fc *fakeRepository) get(cmd *models.GetDataSourceByNameQuery) error {
+	for _, v := range fc.loadAll {
+		if cmd.Name == v.Name && cmd.OrgId == v.OrgId {
+			cmd.Result = v
+			return nil
+		}
+	}
+
+	return models.ErrDataSourceNotFound
+}

pkg/setting/datasources/init.go

@@ -1,114 +0,0 @@
-package datasources
-
-import (
-	"io"
-	"io/ioutil"
-	"path/filepath"
-
-	"github.com/grafana/grafana/pkg/bus"
-	"github.com/grafana/grafana/pkg/log"
-
-	"github.com/grafana/grafana/pkg/models"
-	yaml "gopkg.in/yaml.v2"
-)
-
-var (
-	logger log.Logger
-)
-
-// TODO: secure jsonData
-// TODO: auto reload on file changes
-
-func Init(configPath string) (error, io.Closer) {
-	logger = log.New("setting.datasource")
-
-	datasources, err := readDatasources(configPath)
-	if err != nil {
-		return err, ioutil.NopCloser(nil)
-	}
-
-	for _, ds := range datasources {
-		query := &models.GetDataSourceByNameQuery{Name: ds.Name}
-		err := bus.Dispatch(query)
-		if err != nil && err != models.ErrDataSourceNotFound {
-			return err, ioutil.NopCloser(nil)
-		}
-
-		if ds.OrgId == 0 {
-			ds.OrgId = 1
-		}
-
-		if query.Result == nil {
-			logger.Info("inserting ", "name", ds.Name)
-			insertCmd := insertCommand(ds)
-			if err := bus.Dispatch(insertCmd); err != nil {
-				return err, ioutil.NopCloser(nil)
-			}
-		} else {
-			logger.Info("updating", "name", ds.Name)
-			updateCmd := updateCommand(ds, query.Result.Id)
-			if err := bus.Dispatch(updateCmd); err != nil {
-				return err, ioutil.NopCloser(nil)
-			}
-		}
-	}
-
-	return nil, ioutil.NopCloser(nil)
-}
-
-func readDatasources(path string) ([]models.DataSource, error) {
-	filename, _ := filepath.Abs(path)
-	yamlFile, err := ioutil.ReadFile(filename)
-
-	if err != nil {
-		return nil, err
-	}
-
-	var datasources []models.DataSource
-
-	err = yaml.Unmarshal(yamlFile, &datasources)
-	if err != nil {
-		return nil, err
-	}
-
-	return datasources, nil
-}
-
-func insertCommand(ds models.DataSource) *models.AddDataSourceCommand {
-	return &models.AddDataSourceCommand{
-		OrgId:             ds.OrgId,
-		Name:              ds.Name,
-		Type:              ds.Type,
-		Access:            ds.Access,
-		Url:               ds.Url,
-		Password:          ds.Password,
-		User:              ds.User,
-		Database:          ds.Database,
-		BasicAuth:         ds.BasicAuth,
-		BasicAuthUser:     ds.BasicAuthUser,
-		BasicAuthPassword: ds.BasicAuthPassword,
-		WithCredentials:   ds.WithCredentials,
-		IsDefault:         ds.IsDefault,
-		JsonData:          ds.JsonData,
-	}
-}
-
-func updateCommand(ds models.DataSource, id int64) *models.UpdateDataSourceCommand {
-	return &models.UpdateDataSourceCommand{
-		Id:                id,
-		OrgId:             ds.OrgId,
-		Name:              ds.Name,
-		Type:              ds.Type,
-		Access:            ds.Access,
-		Url:               ds.Url,
-		Password:          ds.Password,
-		User:              ds.User,
-		Database:          ds.Database,
-		BasicAuth:         ds.BasicAuth,
-		BasicAuthUser:     ds.BasicAuthUser,
-		BasicAuthPassword: ds.BasicAuthPassword,
-		WithCredentials:   ds.WithCredentials,
-		IsDefault:         ds.IsDefault,
-		JsonData:          ds.JsonData,
-	}
-}