IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

provisioning: support camcelCase provisioning files

Browse Source
This commit is contained in:
bergquist 2018-02-12 15:17:32 +01:00
parent ead1c300d7
commit 4a35244cb9
9 changed files with 384 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
conf/provisioning/datasources/sample.yaml
View File

@ -1,7 +1,10 @@
# # config file version
# apiVersion: 1
# # list of datasources that should be deleted from the database
#delete_datasources:
#deleteDatasources:
# - name: Graphite
# org_id: 1
# orgId: 1
# # list of datasources to insert/update depending
# # on what's available in the datbase
@ -12,8 +15,8 @@
# type: graphite
# # <string, required> access mode. direct or proxy. Required
# access: proxy
# # <int> org id. will default to org_id 1 if not specified
# org_id: 1
# # <int> org id. will default to orgId 1 if not specified
# orgId: 1
# # <string> url
# url: http://localhost:8080
# # <string> database password, if used
@ -23,22 +26,22 @@
# # <string> database name, if used
# database:
# # <bool> enable/disable basic auth
# basic_auth:
# basicAuth:
# # <string> basic auth username
# basic_auth_user:
# basicAuthUser:
# # <string> basic auth password
# basic_auth_password:
# basicAuthPassword:
# # <bool> enable/disable with credentials headers
# with_credentials:
# withCredentials:
# # <bool> mark as default datasource. Max one per org
# is_default:
# isDefault:
# # <map> fields that will be converted to json and stored in json_data
# json_data:
# jsonData:
# graphiteVersion: "1.1"
# tlsAuth: true
# tlsAuthWithCACert: true
# # <string> json object of data that will be encrypted.
# secure_json_data:
# secureJsonData:
# tlsCACert: "..."
# tlsClientCert: "..."
# tlsClientKey: "..."

27
docs/sources/administration/provisioning.md
View File

@ -81,13 +81,16 @@ If you are running multiple instances of Grafana you might run into problems if
### Example datasource config file
```yaml
# config file version
apiVersion: 1
# list of datasources that should be deleted from the database
delete_datasources:
deleteDatasources:
- name: Graphite
org_id: 1
orgId: 1
# list of datasources to insert/update depending
# whats available in the datbase
# whats available in the database
datasources:
# <string, required> name of the datasource. Required
- name: Graphite
@ -95,8 +98,8 @@ datasources:
type: graphite
# <string, required> access mode. direct or proxy. Required
access: proxy
# <int> org id. will default to org_id 1 if not specified
org_id: 1
# <int> org id. will default to orgId 1 if not specified
orgId: 1
# <string> url
url: http://localhost:8080
# <string> database password, if used
@ -106,22 +109,22 @@ datasources:
# <string> database name, if used
database:
# <bool> enable/disable basic auth
basic_auth:
basicAuth:
# <string> basic auth username
basic_auth_user:
basicAuthUser:
# <string> basic auth password
basic_auth_password:
basicAuthPassword:
# <bool> enable/disable with credentials headers
with_credentials:
withCredentials:
# <bool> mark as default datasource. Max one per org
is_default:
isDefault:
# <map> fields that will be converted to json and stored in json_data
json_data:
jsonData:
graphiteVersion: "1.1"
tlsAuth: true
tlsAuthWithCACert: true
# <string> json object of data that will be encrypted.
secure_json_data:
secureJsonData:
tlsCACert: "..."
tlsClientCert: "..."
tlsClientKey: "..."

105
pkg/services/provisioning/datasources/config_reader.go Normal file
View File

@ -0,0 +1,105 @@
package datasources
import (
"io/ioutil"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/log"
"gopkg.in/yaml.v2"
)
type configReader struct {
log log.Logger
}
func (cr *configReader) readConfig(path string) ([]*DatasourcesAsConfig, error) {
var datasources []*DatasourcesAsConfig
files, err := ioutil.ReadDir(path)
if err != nil {
cr.log.Error("cant read datasource provisioning files from directory", "path", path)
return datasources, nil
}
for _, file := range files {
if strings.HasSuffix(file.Name(), ".yaml") || strings.HasSuffix(file.Name(), ".yml") {
filename, _ := filepath.Abs(filepath.Join(path, file.Name()))
yamlFile, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
datasource, err := parseDatasourceConfig(yamlFile)
if err != nil {
return nil, err
}
if datasource != nil {
datasources = append(datasources, datasource)
}
}
}
err = validateDefaultUniqueness(datasources)
if err != nil {
return nil, err
}
return datasources, nil
}
func parseDatasourceConfig(yamlFile []byte) (*DatasourcesAsConfig, error) {
var apiVersion *ConfigVersion
err := yaml.Unmarshal(yamlFile, &apiVersion)
if err != nil {
return nil, err
}
if apiVersion.ApiVersion > 0 {
var v1 *DatasourcesAsConfigV1
err = yaml.Unmarshal(yamlFile, &v1)
if err != nil {
return nil, err
}
return v1.mapToDatasourceFromConfig(apiVersion.ApiVersion), nil
}
var v0 *DatasourcesAsConfigV0
err = yaml.Unmarshal(yamlFile, &v0)
if err != nil {
return nil, err
}
return v0.mapToDatasourceFromConfig(apiVersion.ApiVersion), nil
}
func validateDefaultUniqueness(datasources []*DatasourcesAsConfig) error {
defaultCount := 0
for i := range datasources {
if datasources[i].Datasources == nil {
continue
}
for _, ds := range datasources[i].Datasources {
if ds.OrgId == 0 {
ds.OrgId = 1
}
if ds.IsDefault {
defaultCount++
if defaultCount > 1 {
return ErrInvalidConfigToManyDefault
}
}
}
for _, ds := range datasources[i].DeleteDatasources {
if ds.OrgId == 0 {
ds.OrgId = 1
}
}
}
return nil
}

80
pkg/services/provisioning/datasources/datasources_test.go → pkg/services/provisioning/datasources/config_reader_test.go
View File

@ -17,6 +17,7 @@ var (
twoDatasourcesConfigPurgeOthers string = "./test-configs/insert-two-delete-two"
doubleDatasourcesConfig string = "./test-configs/double-default"
allProperties string = "./test-configs/all-properties"
versionZero string = "./test-configs/version-0"
brokenYaml string = "./test-configs/broken-yaml"
fakeRepo *fakeRepository
@ -130,7 +131,7 @@ func TestDatasourceAsConfig(t *testing.T) {
So(len(cfg), ShouldEqual, 0)
})
Convey("can read all properties", func() {
Convey("can read all properties from version 1", func() {
cfgProvifer := &configReader{log: log.New("test logger")}
cfg, err := cfgProvifer.readConfig(allProperties)
if err != nil {
@ -140,38 +141,65 @@ func TestDatasourceAsConfig(t *testing.T) {
So(len(cfg), ShouldEqual, 2)
dsCfg := cfg[0]
ds := dsCfg.Datasources[0]
So(ds.Name, ShouldEqual, "name")
So(ds.Type, ShouldEqual, "type")
So(ds.Access, ShouldEqual, models.DS_ACCESS_PROXY)
So(ds.OrgId, ShouldEqual, 2)
So(ds.Url, ShouldEqual, "url")
So(ds.User, ShouldEqual, "user")
So(ds.Password, ShouldEqual, "password")
So(ds.Database, ShouldEqual, "database")
So(ds.BasicAuth, ShouldBeTrue)
So(ds.BasicAuthUser, ShouldEqual, "basic_auth_user")
So(ds.BasicAuthPassword, ShouldEqual, "basic_auth_password")
So(ds.WithCredentials, ShouldBeTrue)
So(ds.IsDefault, ShouldBeTrue)
So(ds.Editable, ShouldBeTrue)
So(dsCfg.ApiVersion, ShouldEqual, 1)
So(len(ds.JsonData), ShouldBeGreaterThan, 2)
So(ds.JsonData["graphiteVersion"], ShouldEqual, "1.1")
So(ds.JsonData["tlsAuth"], ShouldEqual, true)
So(ds.JsonData["tlsAuthWithCACert"], ShouldEqual, true)
validateDatasource(dsCfg)
validateDeleteDatasources(dsCfg)
})
So(len(ds.SecureJsonData), ShouldBeGreaterThan, 2)
So(ds.SecureJsonData["tlsCACert"], ShouldEqual, "MjNOcW9RdkbUDHZmpco2HCYzVq9dE+i6Yi+gmUJotq5CDA==")
So(ds.SecureJsonData["tlsClientCert"], ShouldEqual, "ckN0dGlyMXN503YNfjTcf9CV+GGQneN+xmAclQ==")
So(ds.SecureJsonData["tlsClientKey"], ShouldEqual, "ZkN4aG1aNkja/gKAB1wlnKFIsy2SRDq4slrM0A==")
Convey("can read all properties from version 0", func() {
cfgProvifer := &configReader{log: log.New("test logger")}
cfg, err := cfgProvifer.readConfig(versionZero)
if err != nil {
t.Fatalf("readConfig return an error %v", err)
}
dstwo := cfg[1].Datasources[0]
So(dstwo.Name, ShouldEqual, "name2")
So(len(cfg), ShouldEqual, 1)
dsCfg := cfg[0]
So(dsCfg.ApiVersion, ShouldEqual, 0)
validateDatasource(dsCfg)
validateDeleteDatasources(dsCfg)
})
})
}
func validateDeleteDatasources(dsCfg *DatasourcesAsConfig) {
So(len(dsCfg.DeleteDatasources), ShouldEqual, 1)
deleteDs := dsCfg.DeleteDatasources[0]
So(deleteDs.Name, ShouldEqual, "old-graphite3")
So(deleteDs.OrgId, ShouldEqual, 2)
}
func validateDatasource(dsCfg *DatasourcesAsConfig) {
ds := dsCfg.Datasources[0]
So(ds.Name, ShouldEqual, "name")
So(ds.Type, ShouldEqual, "type")
So(ds.Access, ShouldEqual, models.DS_ACCESS_PROXY)
So(ds.OrgId, ShouldEqual, 2)
So(ds.Url, ShouldEqual, "url")
So(ds.User, ShouldEqual, "user")
So(ds.Password, ShouldEqual, "password")
So(ds.Database, ShouldEqual, "database")
So(ds.BasicAuth, ShouldBeTrue)
So(ds.BasicAuthUser, ShouldEqual, "basic_auth_user")
So(ds.BasicAuthPassword, ShouldEqual, "basic_auth_password")
So(ds.WithCredentials, ShouldBeTrue)
So(ds.IsDefault, ShouldBeTrue)
So(ds.Editable, ShouldBeTrue)
So(ds.Version, ShouldEqual, 10)
So(len(ds.JsonData), ShouldBeGreaterThan, 2)
So(ds.JsonData["graphiteVersion"], ShouldEqual, "1.1")
So(ds.JsonData["tlsAuth"], ShouldEqual, true)
So(ds.JsonData["tlsAuthWithCACert"], ShouldEqual, true)
So(len(ds.SecureJsonData), ShouldBeGreaterThan, 2)
So(ds.SecureJsonData["tlsCACert"], ShouldEqual, "MjNOcW9RdkbUDHZmpco2HCYzVq9dE+i6Yi+gmUJotq5CDA==")
So(ds.SecureJsonData["tlsClientCert"], ShouldEqual, "ckN0dGlyMXN503YNfjTcf9CV+GGQneN+xmAclQ==")
So(ds.SecureJsonData["tlsClientKey"], ShouldEqual, "ZkN4aG1aNkja/gKAB1wlnKFIsy2SRDq4slrM0A==")
}
type fakeRepository struct {
inserted []*models.AddDataSourceCommand

66
pkg/services/provisioning/datasources/datasources.go
View File

@ -2,16 +2,12 @@ package datasources
import (
"errors"
"io/ioutil"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/models"
yaml "gopkg.in/yaml.v2"
)
var (
@ -94,65 +90,3 @@ func (dc *DatasourceProvisioner) deleteDatasources(dsToDelete []*DeleteDatasourc
return nil
}
type configReader struct {
log log.Logger
}
func (cr *configReader) readConfig(path string) ([]*DatasourcesAsConfig, error) {
var datasources []*DatasourcesAsConfig
files, err := ioutil.ReadDir(path)
if err != nil {
cr.log.Error("cant read datasource provisioning files from directory", "path", path)
return datasources, nil
}
for _, file := range files {
if strings.HasSuffix(file.Name(), ".yaml") || strings.HasSuffix(file.Name(), ".yml") {
filename, _ := filepath.Abs(filepath.Join(path, file.Name()))
yamlFile, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
var datasource *DatasourcesAsConfig
err = yaml.Unmarshal(yamlFile, &datasource)
if err != nil {
return nil, err
}
if datasource != nil {
datasources = append(datasources, datasource)
}
}
}
defaultCount := 0
for i := range datasources {
if datasources[i].Datasources == nil {
continue
}
for _, ds := range datasources[i].Datasources {
if ds.OrgId == 0 {
ds.OrgId = 1
}
if ds.IsDefault {
defaultCount++
if defaultCount > 1 {
return nil, ErrInvalidConfigToManyDefault
}
}
}
for _, ds := range datasources[i].DeleteDatasources {
if ds.OrgId == 0 {
ds.OrgId = 1
}
}
}
return datasources, nil
}

23
pkg/services/provisioning/datasources/test-configs/all-properties/all-properties.yaml
View File

@ -1,23 +1,30 @@
apiVersion: 1
datasources:
- name: name
type: type
access: proxy
org_id: 2
orgId: 2
url: url
password: password
user: user
database: database
basic_auth: true
basic_auth_user: basic_auth_user
basic_auth_password: basic_auth_password
with_credentials: true
is_default: true
json_data:
basicAuth: true
basicAuthUser: basic_auth_user
basicAuthPassword: basic_auth_password
withCredentials: true
isDefault: true
jsonData:
graphiteVersion: "1.1"
tlsAuth: true
tlsAuthWithCACert: true
secure_json_data:
secureJsonData:
tlsCACert: "MjNOcW9RdkbUDHZmpco2HCYzVq9dE+i6Yi+gmUJotq5CDA=="
tlsClientCert: "ckN0dGlyMXN503YNfjTcf9CV+GGQneN+xmAclQ=="
tlsClientKey: "ZkN4aG1aNkja/gKAB1wlnKFIsy2SRDq4slrM0A=="
editable: true
version: 10
deleteDatasources:
- name: old-graphite3
orgId: 2

2
pkg/services/provisioning/datasources/test-configs/all-properties/second.yaml
View File

@ -3,5 +3,5 @@ datasources:
- name: name2
type: type2
access: proxy
org_id: 2
orgId: 2
url: url2

28
pkg/services/provisioning/datasources/test-configs/version-0/version-0.yaml Normal file
View File

@ -0,0 +1,28 @@
datasources:
- name: name
type: type
access: proxy
org_id: 2
url: url
password: password
user: user
database: database
basic_auth: true
basic_auth_user: basic_auth_user
basic_auth_password: basic_auth_password
with_credentials: true
is_default: true
json_data:
graphiteVersion: "1.1"
tlsAuth: true
tlsAuthWithCACert: true
secure_json_data:
tlsCACert: "MjNOcW9RdkbUDHZmpco2HCYzVq9dE+i6Yi+gmUJotq5CDA=="
tlsClientCert: "ckN0dGlyMXN503YNfjTcf9CV+GGQneN+xmAclQ=="
tlsClientKey: "ZkN4aG1aNkja/gKAB1wlnKFIsy2SRDq4slrM0A=="
editable: true
version: 10
delete_datasources:
- name: old-graphite3
org_id: 2

158
pkg/services/provisioning/datasources/types.go
View File

@ -1,22 +1,74 @@
package datasources
import "github.com/grafana/grafana/pkg/models"
import (
"github.com/grafana/grafana/pkg/models"
)
import "github.com/grafana/grafana/pkg/components/simplejson"
type ConfigVersion struct {
ApiVersion int64 `json:"apiVersion" yaml:"apiVersion"`
}
type DatasourcesAsConfig struct {
Datasources []*DataSourceFromConfig `json:"datasources" yaml:"datasources"`
DeleteDatasources []*DeleteDatasourceConfig `json:"delete_datasources" yaml:"delete_datasources"`
ApiVersion int64
Datasources []*DataSourceFromConfig
DeleteDatasources []*DeleteDatasourceConfig
}
type DeleteDatasourceConfig struct {
OrgId int64
Name string
}
type DataSourceFromConfig struct {
OrgId int64
Version int
Name string
Type string
Access string
Url string
Password string
User string
Database string
BasicAuth bool
BasicAuthUser string
BasicAuthPassword string
WithCredentials bool
IsDefault bool
JsonData map[string]interface{}
SecureJsonData map[string]string
Editable bool
}
type DatasourcesAsConfigV0 struct {
ConfigVersion
Datasources []*DataSourceFromConfigV0 `json:"datasources" yaml:"datasources"`
DeleteDatasources []*DeleteDatasourceConfigV0 `json:"delete_datasources" yaml:"delete_datasources"`
}
type DatasourcesAsConfigV1 struct {
ConfigVersion
Datasources []*DataSourceFromConfigV1 `json:"datasources" yaml:"datasources"`
DeleteDatasources []*DeleteDatasourceConfigV1 `json:"deleteDatasources" yaml:"deleteDatasources"`
}
type DeleteDatasourceConfigV0 struct {
OrgId int64 `json:"org_id" yaml:"org_id"`
Name string `json:"name" yaml:"name"`
}
type DataSourceFromConfig struct {
OrgId int64 `json:"org_id" yaml:"org_id"`
Version int `json:"version" yaml:"version"`
type DeleteDatasourceConfigV1 struct {
OrgId int64 `json:"orgId" yaml:"orgId"`
Name string `json:"name" yaml:"name"`
}
type DataSourceFromConfigV0 struct {
OrgId int64 `json:"org_id" yaml:"org_id"`
Version int `json:"version" yaml:"version"`
Name string `json:"name" yaml:"name"`
Type string `json:"type" yaml:"type"`
Access string `json:"access" yaml:"access"`
@ -34,6 +86,100 @@ type DataSourceFromConfig struct {
Editable bool `json:"editable" yaml:"editable"`
}
type DataSourceFromConfigV1 struct {
OrgId int64 `json:"orgId" yaml:"orgId"`
Version int `json:"version" yaml:"version"`
Name string `json:"name" yaml:"name"`
Type string `json:"type" yaml:"type"`
Access string `json:"access" yaml:"access"`
Url string `json:"url" yaml:"url"`
Password string `json:"password" yaml:"password"`
User string `json:"user" yaml:"user"`
Database string `json:"database" yaml:"database"`
BasicAuth bool `json:"basicAuth" yaml:"basicAuth"`
BasicAuthUser string `json:"basicAuthUser" yaml:"basicAuthUser"`
BasicAuthPassword string `json:"basicAuthPassword" yaml:"basicAuthPassword"`
WithCredentials bool `json:"withCredentials" yaml:"withCredentials"`
IsDefault bool `json:"isDefault" yaml:"isDefault"`
JsonData map[string]interface{} `json:"jsonData" yaml:"jsonData"`
SecureJsonData map[string]string `json:"secureJsonData" yaml:"secureJsonData"`
Editable bool `json:"editable" yaml:"editable"`
}
func (cfg *DatasourcesAsConfigV1) mapToDatasourceFromConfig(apiVersion int64) *DatasourcesAsConfig {
r := &DatasourcesAsConfig{}
r.ApiVersion = apiVersion
for _, ds := range cfg.Datasources {
r.Datasources = append(r.Datasources, &DataSourceFromConfig{
OrgId: ds.OrgId,
Name: ds.Name,
Type: ds.Type,
Access: ds.Access,
Url: ds.Url,
Password: ds.Password,
User: ds.User,
Database: ds.Database,
BasicAuth: ds.BasicAuth,
BasicAuthUser: ds.BasicAuthUser,
BasicAuthPassword: ds.BasicAuthPassword,
WithCredentials: ds.WithCredentials,
IsDefault: ds.IsDefault,
JsonData: ds.JsonData,
SecureJsonData: ds.SecureJsonData,
Editable: ds.Editable,
Version: ds.Version,
})
}
for _, ds := range cfg.DeleteDatasources {
r.DeleteDatasources = append(r.DeleteDatasources, &DeleteDatasourceConfig{
OrgId: ds.OrgId,
Name: ds.Name,
})
}
return r
}
func (cfg *DatasourcesAsConfigV0) mapToDatasourceFromConfig(apiVersion int64) *DatasourcesAsConfig {
r := &DatasourcesAsConfig{}
r.ApiVersion = apiVersion
for _, ds := range cfg.Datasources {
r.Datasources = append(r.Datasources, &DataSourceFromConfig{
OrgId: ds.OrgId,
Name: ds.Name,
Type: ds.Type,
Access: ds.Access,
Url: ds.Url,
Password: ds.Password,
User: ds.User,
Database: ds.Database,
BasicAuth: ds.BasicAuth,
BasicAuthUser: ds.BasicAuthUser,
BasicAuthPassword: ds.BasicAuthPassword,
WithCredentials: ds.WithCredentials,
IsDefault: ds.IsDefault,
JsonData: ds.JsonData,
SecureJsonData: ds.SecureJsonData,
Editable: ds.Editable,
Version: ds.Version,
})
}
for _, ds := range cfg.DeleteDatasources {
r.DeleteDatasources = append(r.DeleteDatasources, &DeleteDatasourceConfig{
OrgId: ds.OrgId,
Name: ds.Name,
})
}
return r
}
func createInsertCommand(ds *DataSourceFromConfig) *models.AddDataSourceCommand {
jsonData := simplejson.New()
if len(ds.JsonData) > 0 {