datasource as cfg: support globbing

conf/datasources/datasources.yaml

@@ -1,8 +1,3 @@
-# purge data source not listed in configuration.
-# not recommended in HA setups if config can
-# can differ between instances.
-purge_other_datasources: false
-
 # list of datasources to insert/update depending 
 # whats available in the datbase
 datasources:
@@ -42,9 +37,6 @@ datasources:
 #   # <bool> allow users to edit datasources from the UI.
 #   editable: true
 
-
-# - name: Prometheus
-#   type: prometheus
-#   access: proxy
-#   url: http://localhost:9090
-
+delete_datasources:
+  # - name: Graphite
+  #   org_id: 1

conf/defaults.ini

@@ -12,17 +12,17 @@ instance_name = ${HOSTNAME}
 #################################### Paths ###############################
 [paths]
 # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
-#
 data = data
-#
+
 # Directory where grafana can store logs
-#
 logs = data/log
-#
+
 # Directory where grafana will automatically scan and look for plugins
-#
 plugins = data/plugins
 
+# Config files containing datasources that will be configured at startup
+datasources = conf/datasources
+
 #################################### Server ##############################
 [server]
 # Protocol (http, https, socket)

conf/sample.ini

@@ -12,18 +12,17 @@
 #################################### Paths ####################################
 [paths]
 # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
-#
 ;data = /var/lib/grafana
-#
+
 # Directory where grafana can store logs
-#
 ;logs = /var/log/grafana
-#
+
 # Directory where grafana will automatically scan and look for plugins
-#
 ;plugins = /var/lib/grafana/plugins
 
-#
+# Config files containing datasources that will be configured at startup
+;datasources = conf/datasources
+
 #################################### Server ####################################
 [server]
 # Protocol (http, https, socket)

pkg/cmd/grafana-server/server.go

@@ -66,7 +66,8 @@ func (g *GrafanaServerImpl) Start() {
 	social.NewOAuthService()
 	plugins.Init()
 
-	if err := provisioning.StartUp(setting.HomePath); err != nil {
+	//if err := provisioning.StartUp(setting.HomePath); err != nil {
+	if err := provisioning.StartUp(setting.DatasourcesPath); err != nil {
 		logger.Error("Failed to provision Grafana from config", "error", err)
 		g.Shutdown(1, "Startup failed")
 		return

pkg/models/datasource.go

@@ -146,11 +146,15 @@ type UpdateDataSourceCommand struct {
 type DeleteDataSourceByIdCommand struct {
 	Id    int64
 	OrgId int64
+
+	DeletedDatasourcesCount int64
 }
 
 type DeleteDataSourceByNameCommand struct {
 	Name  string
 	OrgId int64
+
+	DeletedDatasourcesCount int64
 }
 
 // ---------------------

pkg/services/provisioning/datasources/datasources.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"io/ioutil"
 	"path/filepath"
+	"strings"
 
 	"github.com/grafana/grafana/pkg/bus"
 
@@ -17,67 +18,36 @@ var (
 	ErrInvalidConfigToManyDefault = errors.New("datasource.yaml config is invalid. Only one datasource can be marked as default")
 )
 
-func Apply(configPath string) error {
-	dc := NewDatasourceConfiguration()
-	return dc.applyChanges(configPath)
+func Provision(configDirectory string) error {
+	dc := newDatasourceProvisioner(log.New("provisioning.datasources"))
+	return dc.applyChanges(configDirectory)
 }
 
-type DatasourceConfigurator struct {
+type DatasourceProvisioner struct {
 	log         log.Logger
-	cfgProvider configProvider
+	cfgProvider configReader
 }
 
-func NewDatasourceConfiguration() DatasourceConfigurator {
-	return newDatasourceConfiguration(log.New("setting.datasource"))
-}
-
-func newDatasourceConfiguration(log log.Logger) DatasourceConfigurator {
-	return DatasourceConfigurator{
+func newDatasourceProvisioner(log log.Logger) DatasourceProvisioner {
+	return DatasourceProvisioner{
 		log:         log,
-		cfgProvider: configProvider{},
+		cfgProvider: configReader{},
 	}
 }
 
-func (dc *DatasourceConfigurator) applyChanges(configPath string) error {
-	cfg, err := dc.cfgProvider.readConfig(configPath)
-	if err != nil {
-		return err
-	}
-
-	defaultCount := 0
-	for i := range cfg.Datasources {
-		if cfg.Datasources[i].OrgId == 0 {
-			cfg.Datasources[i].OrgId = 1
-		}
-
-		if cfg.Datasources[i].IsDefault {
-			defaultCount++
-			if defaultCount > 1 {
-				return ErrInvalidConfigToManyDefault
-			}
-		}
-	}
-
-	cmd := &models.GetAllDataSourcesQuery{}
-	if err = bus.Dispatch(cmd); err != nil {
-		return err
-	}
-	allDatasources := cmd.Result
-
-	if err := dc.deleteDatasourcesNotInConfiguration(cfg, allDatasources); err != nil {
+func (dc *DatasourceProvisioner) apply(cfg *DatasourcesAsConfig) error {
+	if err := dc.deleteDatasources(cfg.DeleteDatasources); err != nil {
 		return err
 	}
 
 	for _, ds := range cfg.Datasources {
-		var dbDatasource *models.DataSource
-		for _, ddd := range allDatasources {
-			if ddd.Name == ds.Name && ddd.OrgId == ds.OrgId {
-				dbDatasource = ddd
-				break
-			}
+		cmd := &models.GetDataSourceByNameQuery{OrgId: ds.OrgId, Name: ds.Name}
+		err := bus.Dispatch(cmd)
+		if err != nil && err != models.ErrDataSourceNotFound {
+			return err
 		}
 
-		if dbDatasource == nil {
+		if err == models.ErrDataSourceNotFound {
 			dc.log.Info("inserting datasource from configuration ", "name", ds.Name)
 			insertCmd := createInsertCommand(ds)
 			if err := bus.Dispatch(insertCmd); err != nil {
@@ -85,7 +55,7 @@ func (dc *DatasourceConfigurator) applyChanges(configPath string) error {
 			}
 		} else {
 			dc.log.Debug("updating datasource from configuration", "name", ds.Name)
-			updateCmd := createUpdateCommand(ds, dbDatasource.Id)
+			updateCmd := createUpdateCommand(ds, cmd.Result.Id)
 			if err := bus.Dispatch(updateCmd); err != nil {
 				return err
 			}
@@ -95,44 +65,83 @@ func (dc *DatasourceConfigurator) applyChanges(configPath string) error {
 	return nil
 }
 
-func (dc *DatasourceConfigurator) deleteDatasourcesNotInConfiguration(cfg *DatasourcesAsConfig, allDatasources []*models.DataSource) error {
-	if cfg.PurgeOtherDatasources {
-		for _, dbDS := range allDatasources {
-			delete := true
-			for _, cfgDS := range cfg.Datasources {
-				if dbDS.Name == cfgDS.Name && dbDS.OrgId == cfgDS.OrgId {
-					delete = false
-				}
-			}
+func (dc *DatasourceProvisioner) applyChanges(configPath string) error {
+	configs, err := dc.cfgProvider.readConfig(configPath)
+	if err != nil {
+		return err
+	}
 
-			if delete {
-				dc.log.Info("deleting datasource from configuration", "name", dbDS.Name)
-				cmd := &models.DeleteDataSourceByIdCommand{Id: dbDS.Id, OrgId: dbDS.OrgId}
-				if err := bus.Dispatch(cmd); err != nil {
-					return err
-				}
-			}
+	for _, cfg := range configs {
+		if err := dc.apply(cfg); err != nil {
+			return err
 		}
 	}
 
 	return nil
 }
 
-type configProvider struct{}
+func (dc *DatasourceProvisioner) deleteDatasources(dsToDelete []*DeleteDatasourceConfig) error {
+	for _, ds := range dsToDelete {
+		cmd := &models.DeleteDataSourceByNameCommand{OrgId: ds.OrgId, Name: ds.Name}
+		if err := bus.Dispatch(cmd); err != nil {
+			return err
+		}
 
-func (configProvider) readConfig(path string) (*DatasourcesAsConfig, error) {
-	filename, _ := filepath.Abs(path)
-	yamlFile, err := ioutil.ReadFile(filename)
+		if cmd.DeletedDatasourcesCount > 0 {
+			dc.log.Info("deleted datasource based on configuration", "name", ds.Name)
+		}
+	}
 
+	return nil
+}
+
+type configReader struct{}
+
+func (configReader) readConfig(path string) ([]*DatasourcesAsConfig, error) {
+	files, err := ioutil.ReadDir(path)
 	if err != nil {
 		return nil, err
 	}
 
-	var datasources *DatasourcesAsConfig
+	var datasources []*DatasourcesAsConfig
+	for _, file := range files {
+		if strings.HasSuffix(file.Name(), ".yaml") || strings.HasSuffix(file.Name(), ".yml") {
+			filename, _ := filepath.Abs(filepath.Join(path, file.Name()))
+			yamlFile, err := ioutil.ReadFile(filename)
 
-	err = yaml.Unmarshal(yamlFile, &datasources)
-	if err != nil {
-		return nil, err
+			if err != nil {
+				return nil, err
+			}
+			var datasource *DatasourcesAsConfig
+			err = yaml.Unmarshal(yamlFile, &datasource)
+			if err != nil {
+				return nil, err
+			}
+
+			datasources = append(datasources, datasource)
+		}
+	}
+
+	defaultCount := 0
+	for _, cfg := range datasources {
+		for _, ds := range cfg.Datasources {
+			if ds.OrgId == 0 {
+				ds.OrgId = 1
+			}
+
+			if ds.IsDefault {
+				defaultCount++
+				if defaultCount > 1 {
+					return nil, ErrInvalidConfigToManyDefault
+				}
+			}
+		}
+
+		for _, ds := range cfg.DeleteDatasources {
+			if ds.OrgId == 0 {
+				ds.OrgId = 1
+			}
+		}
 	}
 
 	return datasources, nil

pkg/services/provisioning/datasources/datasources_test.go

@@ -13,10 +13,11 @@ import (
 var (
 	logger                          log.Logger = log.New("fake.logger")
 	oneDatasourcesConfig            string     = ""
-	twoDatasourcesConfig            string     = "./test-configs/two-datasources.yaml"
-	twoDatasourcesConfigPurgeOthers string     = "./test-configs/two-datasources-purge-others.yaml"
-	doubleDatasourcesConfig         string     = "./test-configs/double-default-datasources.yaml"
-	allProperties                   string     = "./test-configs/all-properties.yaml"
+	twoDatasourcesConfig            string     = "./test-configs/two-datasources"
+	twoDatasourcesConfigPurgeOthers string     = "./test-configs/insert-two-delete-two"
+	doubleDatasourcesConfig         string     = "./test-configs/double-default"
+	allProperties                   string     = "./test-configs/all-properties"
+	brokenYaml                      string     = "./test-configs/broken-yaml"
 
 	fakeRepo *fakeRepository
 )
@@ -33,7 +34,7 @@ func TestDatasourceAsConfig(t *testing.T) {
 
 		Convey("One configured datasource", func() {
 			Convey("no datasource in database", func() {
-				dc := newDatasourceConfiguration(logger)
+				dc := newDatasourceProvisioner(logger)
 				err := dc.applyChanges(twoDatasourcesConfig)
 				if err != nil {
 					t.Fatalf("applyChanges return an error %v", err)
@@ -50,7 +51,7 @@ func TestDatasourceAsConfig(t *testing.T) {
 				}
 
 				Convey("should update one datasource", func() {
-					dc := newDatasourceConfiguration(logger)
+					dc := newDatasourceProvisioner(logger)
 					err := dc.applyChanges(twoDatasourcesConfig)
 					if err != nil {
 						t.Fatalf("applyChanges return an error %v", err)
@@ -63,7 +64,7 @@ func TestDatasourceAsConfig(t *testing.T) {
 			})
 
 			Convey("Two datasources with is_default", func() {
-				dc := newDatasourceConfiguration(logger)
+				dc := newDatasourceProvisioner(logger)
 				err := dc.applyChanges(doubleDatasourcesConfig)
 				Convey("should raise error", func() {
 					So(err, ShouldEqual, ErrInvalidConfigToManyDefault)
@@ -79,7 +80,7 @@ func TestDatasourceAsConfig(t *testing.T) {
 				}
 
 				Convey("should have two new datasources", func() {
-					dc := newDatasourceConfiguration(logger)
+					dc := newDatasourceProvisioner(logger)
 					err := dc.applyChanges(twoDatasourcesConfigPurgeOthers)
 					if err != nil {
 						t.Fatalf("applyChanges return an error %v", err)
@@ -100,7 +101,7 @@ func TestDatasourceAsConfig(t *testing.T) {
 				}
 
 				Convey("should have two new datasources", func() {
-					dc := newDatasourceConfiguration(logger)
+					dc := newDatasourceProvisioner(logger)
 					err := dc.applyChanges(twoDatasourcesConfig)
 					if err != nil {
 						t.Fatalf("applyChanges return an error %v", err)
@@ -113,16 +114,22 @@ func TestDatasourceAsConfig(t *testing.T) {
 			})
 		})
 
-		Convey("can read all properties", func() {
+		Convey("broken yaml should return error", func() {
+			_, err := configReader{}.readConfig(brokenYaml)
+			So(err, ShouldNotBeNil)
+		})
 
-			cfgProvifer := configProvider{}
+		Convey("can read all properties", func() {
+			cfgProvifer := configReader{}
 			cfg, err := cfgProvifer.readConfig(allProperties)
 			if err != nil {
 				t.Fatalf("readConfig return an error %v", err)
 			}
 
-			So(cfg.PurgeOtherDatasources, ShouldBeTrue)
-			ds := cfg.Datasources[0]
+			So(len(cfg), ShouldEqual, 2)
+
+			dsCfg := cfg[0]
+			ds := dsCfg.Datasources[0]
 
 			So(ds.Name, ShouldEqual, "name")
 			So(ds.Type, ShouldEqual, "type")
@@ -138,19 +145,22 @@ func TestDatasourceAsConfig(t *testing.T) {
 			So(ds.WithCredentials, ShouldBeTrue)
 			So(ds.IsDefault, ShouldBeTrue)
 			So(ds.Editable, ShouldBeTrue)
+
+			dstwo := cfg[1].Datasources[0]
+			So(dstwo.Name, ShouldEqual, "name2")
 		})
 	})
 }
 
 type fakeRepository struct {
 	inserted []*models.AddDataSourceCommand
-	deleted  []*models.DeleteDataSourceByIdCommand
+	deleted  []*models.DeleteDataSourceByNameCommand
 	updated  []*models.UpdateDataSourceCommand
 
 	loadAll []*models.DataSource
 }
 
-func mockDelete(cmd *models.DeleteDataSourceByIdCommand) error {
+func mockDelete(cmd *models.DeleteDataSourceByNameCommand) error {
 	fakeRepo.deleted = append(fakeRepo.deleted, cmd)
 	return nil
 }

pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml

@@ -1,4 +1,3 @@
-purge_other_datasources: true
 datasources:
   - name: name
     type: type

pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml

@@ -0,0 +1,7 @@
+purge_other_datasources: true
+datasources:
+  - name: name2
+    type: type2
+    access: proxy
+    org_id: 2
+    url: url2

pkg/services/provisioning/datasources/test-configs/broken-yaml/broken.yaml

@@ -0,0 +1,6 @@
+#sfxzgnsxzcvnbzcvn
+cvbn
+cvbn
+c
+vbn
+cvbncvbn

pkg/services/provisioning/datasources/test-configs/double-default-datasources.yaml

@@ -1,12 +0,0 @@
-purge_other_datasources: false
-datasources:
-  - name: Graphite
-    type: graphite
-    access: proxy
-    url: http://localhost:8080
-    is_default: true
-  - name: Prometheus
-    type: prometheus
-    access: proxy
-    url: http://localhost:9090
-    is_default: true

pkg/services/provisioning/datasources/test-configs/double-default/default-1.yaml

@@ -0,0 +1,7 @@
+datasources:
+  - name: Graphite
+    type: graphite
+    access: proxy
+    url: http://localhost:8080
+    is_default: true
+

pkg/services/provisioning/datasources/test-configs/double-default/default-2.yaml

@@ -0,0 +1,7 @@
+datasources:
+  - name: Graphite
+    type: graphite
+    access: proxy
+    url: http://localhost:8080
+    is_default: true
+

pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/one-datasources.yaml

@@ -0,0 +1,7 @@
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://localhost:9090
+delete_datasources:
+  - name: old-graphite

pkg/services/provisioning/datasources/test-configs/insert-two-delete-two/two-datasources.yml

@@ -0,0 +1,7 @@
+datasources:
+  - name: Graphite
+    type: graphite
+    access: proxy
+    url: http://localhost:8080
+delete_datasources:
+  - name: old-graphite3

pkg/services/provisioning/datasources/test-configs/two-datasources-purge-others.yaml

@@ -1,10 +0,0 @@
-purge_other_datasources: true
-datasources:
-  - name: Graphite
-    type: graphite
-    access: proxy
-    url: http://localhost:8080
-  - name: Prometheus
-    type: prometheus
-    access: proxy
-    url: http://localhost:9090

pkg/services/provisioning/datasources/test-configs/two-datasources/two-datasources.yaml

@@ -1,4 +1,3 @@
-purge_other_datasources: false
 datasources:
   - name: Graphite
     type: graphite

pkg/services/provisioning/datasources/test-configs/zero-datasources.yaml

@@ -1,2 +0,0 @@
-purge_other_datasources: false
-datasources:

pkg/services/provisioning/datasources/types.go

@@ -4,8 +4,13 @@ import "github.com/grafana/grafana/pkg/models"
 import "github.com/grafana/grafana/pkg/components/simplejson"
 
 type DatasourcesAsConfig struct {
-	PurgeOtherDatasources bool                   `json:"purge_other_datasources" yaml:"purge_other_datasources"`
-	Datasources           []DataSourceFromConfig `json:"datasources" yaml:"datasources"`
+	Datasources       []*DataSourceFromConfig   `json:"datasources" yaml:"datasources"`
+	DeleteDatasources []*DeleteDatasourceConfig `json:"delete_datasources" yaml:"delete_datasources"`
+}
+
+type DeleteDatasourceConfig struct {
+	OrgId int64  `json:"org_id" yaml:"org_id"`
+	Name  string `json:"name" yaml:"name"`
 }
 
 type DataSourceFromConfig struct {
@@ -29,7 +34,7 @@ type DataSourceFromConfig struct {
 	Editable          bool              `json:"editable" yaml:"editable"`
 }
 
-func createInsertCommand(ds DataSourceFromConfig) *models.AddDataSourceCommand {
+func createInsertCommand(ds *DataSourceFromConfig) *models.AddDataSourceCommand {
 	jsonData, err := simplejson.NewJson([]byte(ds.JsonData))
 	if err != nil {
 		jsonData = simplejson.New()
@@ -55,7 +60,7 @@ func createInsertCommand(ds DataSourceFromConfig) *models.AddDataSourceCommand {
 	}
 }
 
-func createUpdateCommand(ds DataSourceFromConfig, id int64) *models.UpdateDataSourceCommand {
+func createUpdateCommand(ds *DataSourceFromConfig, id int64) *models.UpdateDataSourceCommand {
 	jsonData, err := simplejson.NewJson([]byte(ds.JsonData))
 	if err != nil {
 		jsonData = simplejson.New()

pkg/services/provisioning/provisioning.go

@@ -1,8 +1,6 @@
 package provisioning
 
 import (
-	"path/filepath"
-
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/services/provisioning/datasources"
 )
@@ -11,6 +9,6 @@ var (
 	logger log.Logger = log.New("services.provisioning")
 )
 
-func StartUp(homePath string) error {
-	return datasources.Apply(filepath.Join(homePath, "conf/datasources.yaml"))
+func StartUp(datasourcePath string) error {
+	return datasources.Provision(datasourcePath)
 }

pkg/services/sqlstore/datasource.go

@@ -65,7 +65,9 @@ func GetAllDataSources(query *m.GetAllDataSourcesQuery) error {
 func DeleteDataSourceById(cmd *m.DeleteDataSourceByIdCommand) error {
 	return inTransaction(func(sess *DBSession) error {
 		var rawSql = "DELETE FROM data_source WHERE id=? and org_id=?"
-		_, err := sess.Exec(rawSql, cmd.Id, cmd.OrgId)
+		result, err := sess.Exec(rawSql, cmd.Id, cmd.OrgId)
+		affected, _ := result.RowsAffected()
+		cmd.DeletedDatasourcesCount = affected
 		return err
 	})
 }
@@ -73,7 +75,9 @@ func DeleteDataSourceById(cmd *m.DeleteDataSourceByIdCommand) error {
 func DeleteDataSourceByName(cmd *m.DeleteDataSourceByNameCommand) error {
 	return inTransaction(func(sess *DBSession) error {
 		var rawSql = "DELETE FROM data_source WHERE name=? and org_id=?"
-		_, err := sess.Exec(rawSql, cmd.Name, cmd.OrgId)
+		result, err := sess.Exec(rawSql, cmd.Name, cmd.OrgId)
+		affected, _ := result.RowsAffected()
+		cmd.DeletedDatasourcesCount = affected
 		return err
 	})
 }

pkg/setting/setting.go

@@ -50,11 +50,12 @@ var (
 	BuildStamp   int64
 
 	// Paths
-	LogsPath       string
-	HomePath       string
-	DataPath       string
-	PluginsPath    string
-	CustomInitPath = "conf/custom.ini"
+	LogsPath        string
+	HomePath        string
+	DataPath        string
+	PluginsPath     string
+	DatasourcesPath string
+	CustomInitPath  = "conf/custom.ini"
 
 	// Log settings.
 	LogModes   []string
@@ -470,6 +471,7 @@ func NewConfigContext(args *CommandLineArgs) error {
 	Env = Cfg.Section("").Key("app_mode").MustString("development")
 	InstanceName = Cfg.Section("").Key("instance_name").MustString("unknown_instance_name")
 	PluginsPath = makeAbsolute(Cfg.Section("paths").Key("plugins").String(), HomePath)
+	DatasourcesPath = makeAbsolute(Cfg.Section("paths").Key("datasources").String(), HomePath)
 
 	server := Cfg.Section("server")
 	AppUrl, AppSubUrl = parseAppUrlAndSubUrl(server)
@@ -661,5 +663,6 @@ func LogConfigurationInfo() {
 	logger.Info("Path Data", "path", DataPath)
 	logger.Info("Path Logs", "path", LogsPath)
 	logger.Info("Path Plugins", "path", PluginsPath)
+	logger.Info("Path Datasources", "path", DatasourcesPath)
 	logger.Info("App mode " + Env)
 }