datasource as cfg: add support for securedata field

pkg/setting/datasources/datasources.go

@@ -11,13 +11,9 @@ import (
 	yaml "gopkg.in/yaml.v2"
 )
 
-// TODO: secure jsonData
-// TODO: auto reload on file changes
-// TODO: remove get method since all datasources is in memory
-
 type DatasourcesAsConfig struct {
 	PurgeOtherDatasources bool
-	Datasources           []models.DataSource
+	Datasources           []DataSourceFromConfig
 }
 
 func Init(configPath string) error {
@@ -49,50 +45,39 @@ func (dc *DatasourceConfigurator) applyChanges(configPath string) error {
 		return err
 	}
 
-	all, err := dc.repository.loadAllDatasources()
+	allDatasources, err := dc.repository.loadAllDatasources()
 	if err != nil {
 		return err
 	}
 
-	for i, _ := range cfg.Datasources {
+	for i := range cfg.Datasources {
 		if cfg.Datasources[i].OrgId == 0 {
 			cfg.Datasources[i].OrgId = 1
 		}
 	}
 
-	if cfg.PurgeOtherDatasources {
+	if err := dc.deleteDatasourcesNotInConfiguration(cfg, allDatasources); err != nil {
-		for _, dbDatasource := range all {
+		return err
-			delete := true
-			for _, cfgDatasource := range cfg.Datasources {
-				if dbDatasource.Name == cfgDatasource.Name && dbDatasource.OrgId == cfgDatasource.OrgId {
-					delete = false
-				}
-			}
-
-			if delete {
-				dc.log.Info("deleting datasource since PurgeOtherDatasource is enabled", "name", dbDatasource.Name)
-				dc.repository.delete(&models.DeleteDataSourceByIdCommand{Id: dbDatasource.Id, OrgId: dbDatasource.OrgId})
-			}
-		}
 	}
 
 	for _, ds := range cfg.Datasources {
-
+		var dbDatasource *models.DataSource
-		query := &models.GetDataSourceByNameQuery{Name: ds.Name, OrgId: ds.OrgId}
+		for _, ddd := range allDatasources {
-		err := dc.repository.get(query)
+			if ddd.Name == ds.Name && ddd.OrgId == ds.OrgId {
-		if err != nil && err != models.ErrDataSourceNotFound {
+				dbDatasource = ddd
-			return err
+				break
+			}
 		}
 
-		if query.Result == nil {
+		if dbDatasource == nil {
-			dc.log.Info("inserting ", "name", ds.Name)
+			dc.log.Info("inserting datasource from configuration ", "name", ds.Name)
 			insertCmd := createInsertCommand(ds)
 			if err := dc.repository.insert(insertCmd); err != nil {
 				return err
 			}
 		} else {
-			dc.log.Info("updating", "name", ds.Name)
+			dc.log.Info("updating datasource from configuration", "name", ds.Name)
-			updateCmd := createUpdateCommand(ds, query.Result.Id)
+			updateCmd := createUpdateCommand(ds, dbDatasource.Id)
 			if err := dc.repository.update(updateCmd); err != nil {
 				return err
 			}
@@ -102,43 +87,27 @@ func (dc *DatasourceConfigurator) applyChanges(configPath string) error {
 	return nil
 }
 
-func createInsertCommand(ds models.DataSource) *models.AddDataSourceCommand {
+func (dc *DatasourceConfigurator) deleteDatasourcesNotInConfiguration(cfg *DatasourcesAsConfig, allDatasources []*models.DataSource) error {
-	return &models.AddDataSourceCommand{
+	if cfg.PurgeOtherDatasources {
-		OrgId:             ds.OrgId,
+		for _, dbDS := range allDatasources {
-		Name:              ds.Name,
+			delete := true
-		Type:              ds.Type,
+			for _, cfgDS := range cfg.Datasources {
-		Access:            ds.Access,
+				if dbDS.Name == cfgDS.Name && dbDS.OrgId == cfgDS.OrgId {
-		Url:               ds.Url,
+					delete = false
-		Password:          ds.Password,
+				}
-		User:              ds.User,
+			}
-		Database:          ds.Database,
-		BasicAuth:         ds.BasicAuth,
-		BasicAuthUser:     ds.BasicAuthUser,
-		BasicAuthPassword: ds.BasicAuthPassword,
-		WithCredentials:   ds.WithCredentials,
-		IsDefault:         ds.IsDefault,
-		JsonData:          ds.JsonData,
-	}
-}
 
-func createUpdateCommand(ds models.DataSource, id int64) *models.UpdateDataSourceCommand {
+			if delete {
-	return &models.UpdateDataSourceCommand{
+				dc.log.Info("deleting datasource from configuration", "name", dbDS.Name)
-		Id:                id,
+				cmd := &models.DeleteDataSourceByIdCommand{Id: dbDS.Id, OrgId: dbDS.OrgId}
-		OrgId:             ds.OrgId,
+				if err := dc.repository.delete(cmd); err != nil {
-		Name:              ds.Name,
+					return err
-		Type:              ds.Type,
+				}
-		Access:            ds.Access,
+			}
-		Url:               ds.Url,
+		}
-		Password:          ds.Password,
-		User:              ds.User,
-		Database:          ds.Database,
-		BasicAuth:         ds.BasicAuth,
-		BasicAuthUser:     ds.BasicAuthUser,
-		BasicAuthPassword: ds.BasicAuthPassword,
-		WithCredentials:   ds.WithCredentials,
-		IsDefault:         ds.IsDefault,
-		JsonData:          ds.JsonData,
 	}
+
+	return nil
 }
 
 type datasourceRepository interface {

pkg/setting/datasources/datasources_test.go

@@ -19,8 +19,8 @@ func TestDatasourceAsConfig(t *testing.T) {
 		Convey("One configured datasource", func() {
 			fakeCfg.cfg = &DatasourcesAsConfig{
 				PurgeOtherDatasources: false,
-				Datasources: []models.DataSource{
+				Datasources: []DataSourceFromConfig{
-					models.DataSource{Name: "graphite", OrgId: 1},
+					{Name: "graphite", OrgId: 1},
 				},
 			}
 
@@ -38,7 +38,7 @@ func TestDatasourceAsConfig(t *testing.T) {
 
 			Convey("One datasource in database with same name", func() {
 				fakeRepo.loadAll = []*models.DataSource{
-					&models.DataSource{Name: "graphite", OrgId: 1, Id: 1},
+					{Name: "graphite", OrgId: 1, Id: 1},
 				}
 
 				Convey("should update one datasource", func() {
@@ -56,7 +56,7 @@ func TestDatasourceAsConfig(t *testing.T) {
 
 			Convey("One datasource in database with new name", func() {
 				fakeRepo.loadAll = []*models.DataSource{
-					&models.DataSource{Name: "old-graphite", OrgId: 1, Id: 1},
+					{Name: "old-graphite", OrgId: 1, Id: 1},
 				}
 
 				Convey("should update one datasource", func() {
@@ -76,16 +76,16 @@ func TestDatasourceAsConfig(t *testing.T) {
 		Convey("Two configured datasource and purge others ", func() {
 			fakeCfg.cfg = &DatasourcesAsConfig{
 				PurgeOtherDatasources: true,
-				Datasources: []models.DataSource{
+				Datasources: []DataSourceFromConfig{
-					models.DataSource{Name: "graphite", OrgId: 1},
+					{Name: "graphite", OrgId: 1},
-					models.DataSource{Name: "prometheus", OrgId: 1},
+					{Name: "prometheus", OrgId: 1},
 				},
 			}
 
 			Convey("two other datasources in database", func() {
 				fakeRepo.loadAll = []*models.DataSource{
-					&models.DataSource{Name: "old-graphite", OrgId: 1, Id: 1},
+					{Name: "old-graphite", OrgId: 1, Id: 1},
-					&models.DataSource{Name: "old-graphite2", OrgId: 1, Id: 2},
+					{Name: "old-graphite2", OrgId: 1, Id: 2},
 				}
 
 				Convey("should have two new datasources", func() {
@@ -100,22 +100,21 @@ func TestDatasourceAsConfig(t *testing.T) {
 					So(len(fakeRepo.updated), ShouldEqual, 0)
 				})
 			})
-
 		})
 
 		Convey("Two configured datasource and purge others = false", func() {
 			fakeCfg.cfg = &DatasourcesAsConfig{
 				PurgeOtherDatasources: false,
-				Datasources: []models.DataSource{
+				Datasources: []DataSourceFromConfig{
-					models.DataSource{Name: "graphite", OrgId: 1},
+					{Name: "graphite", OrgId: 1},
-					models.DataSource{Name: "prometheus", OrgId: 1},
+					{Name: "prometheus", OrgId: 1},
 				},
 			}
 
 			Convey("two other datasources in database", func() {
 				fakeRepo.loadAll = []*models.DataSource{
-					&models.DataSource{Name: "old-graphite", OrgId: 1, Id: 1},
+					{Name: "graphite", OrgId: 1, Id: 1},
-					&models.DataSource{Name: "old-graphite2", OrgId: 1, Id: 2},
+					{Name: "old-graphite2", OrgId: 1, Id: 2},
 				}
 
 				Convey("should have two new datasources", func() {
@@ -126,11 +125,10 @@ func TestDatasourceAsConfig(t *testing.T) {
 					}
 
 					So(len(fakeRepo.deleted), ShouldEqual, 0)
-					So(len(fakeRepo.inserted), ShouldEqual, 2)
+					So(len(fakeRepo.inserted), ShouldEqual, 1)
-					So(len(fakeRepo.updated), ShouldEqual, 0)
+					So(len(fakeRepo.updated), ShouldEqual, 1)
 				})
 			})
-
 		})
 	})
 }

pkg/setting/datasources/types.go

@@ -0,0 +1,76 @@
+package datasources
+
+import "github.com/grafana/grafana/pkg/models"
+import "github.com/grafana/grafana/pkg/components/simplejson"
+
+type DataSourceFromConfig struct {
+	Id      int64
+	OrgId   int64
+	Version int
+
+	Name              string
+	Type              string
+	Access            string
+	Url               string
+	Password          string
+	User              string
+	Database          string
+	BasicAuth         bool
+	BasicAuthUser     string
+	BasicAuthPassword string
+	WithCredentials   bool
+	IsDefault         bool
+	JsonData          string
+	SecureJsonData    map[string]string
+}
+
+func createInsertCommand(ds DataSourceFromConfig) *models.AddDataSourceCommand {
+	jsonData, err := simplejson.NewJson([]byte(ds.JsonData))
+	if err != nil {
+		jsonData = simplejson.New()
+	}
+
+	return &models.AddDataSourceCommand{
+		OrgId:             ds.OrgId,
+		Name:              ds.Name,
+		Type:              ds.Type,
+		Access:            models.DsAccess(ds.Access),
+		Url:               ds.Url,
+		Password:          ds.Password,
+		User:              ds.User,
+		Database:          ds.Database,
+		BasicAuth:         ds.BasicAuth,
+		BasicAuthUser:     ds.BasicAuthUser,
+		BasicAuthPassword: ds.BasicAuthPassword,
+		WithCredentials:   ds.WithCredentials,
+		IsDefault:         ds.IsDefault,
+		JsonData:          jsonData,
+		SecureJsonData:    ds.SecureJsonData,
+	}
+}
+
+func createUpdateCommand(ds DataSourceFromConfig, id int64) *models.UpdateDataSourceCommand {
+	jsonData, err := simplejson.NewJson([]byte(ds.JsonData))
+	if err != nil {
+		jsonData = simplejson.New()
+	}
+
+	return &models.UpdateDataSourceCommand{
+		Id:                id,
+		OrgId:             ds.OrgId,
+		Name:              ds.Name,
+		Type:              ds.Type,
+		Access:            models.DsAccess(ds.Access),
+		Url:               ds.Url,
+		Password:          ds.Password,
+		User:              ds.User,
+		Database:          ds.Database,
+		BasicAuth:         ds.BasicAuth,
+		BasicAuthUser:     ds.BasicAuthUser,
+		BasicAuthPassword: ds.BasicAuthPassword,
+		WithCredentials:   ds.WithCredentials,
+		IsDefault:         ds.IsDefault,
+		JsonData:          jsonData,
+		SecureJsonData:    ds.SecureJsonData,
+	}
+}