Provisioning: Allows specifying uid for datasource and use that in derived fields (#23585)

* Add uid to datasource

* Fix uid passing when provisioning

* Better error handling and Uid column type change

* Fix test and strict null error counts

* Add backend tests

* Add tests

* Fix strict null checks

* Update test

* Improve tests

* Update pkg/services/sqlstore/datasource.go

Co-Authored-By: Arve Knudsen <arve.knudsen@gmail.com>

* Variable rename

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
This commit is contained in:
Andrej Ocenas
2020-04-20 15:48:38 +02:00
committed by GitHub
parent d5f8d976f0
commit e5dd7efdee
34 changed files with 446 additions and 204 deletions

View File

@@ -128,7 +128,7 @@ func AddDataSource(c *models.ReqContext, cmd models.AddDataSourceCommand) Respon
cmd.OrgId = c.OrgId
if err := bus.Dispatch(&cmd); err != nil {
if err == models.ErrDataSourceNameExists {
if err == models.ErrDataSourceNameExists || err == models.ErrDataSourceUidExists {
return Error(409, err.Error(), err)
}

View File

@@ -67,6 +67,7 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i
var dsMap = map[string]interface{}{
"id": ds.Id,
"uid": ds.Uid,
"type": ds.Type,
"name": ds.Name,
"url": url,

View File

@@ -19,10 +19,10 @@ func TestPasswordMigrationCommand(t *testing.T) {
defer session.Close()
datasources := []*models.DataSource{
{Type: "influxdb", Name: "influxdb", Password: "foobar"},
{Type: "graphite", Name: "graphite", BasicAuthPassword: "foobar"},
{Type: "prometheus", Name: "prometheus"},
{Type: "elasticsearch", Name: "elasticsearch", Password: "pwd"},
{Type: "influxdb", Name: "influxdb", Password: "foobar", Uid: "influx"},
{Type: "graphite", Name: "graphite", BasicAuthPassword: "foobar", Uid: "graphite"},
{Type: "prometheus", Name: "prometheus", Uid: "prom"},
{Type: "elasticsearch", Name: "elasticsearch", Password: "pwd", Uid: "elastic"},
}
// set required default values

View File

@@ -28,11 +28,13 @@ const (
)
var (
ErrDataSourceNotFound = errors.New("Data source not found")
ErrDataSourceNameExists = errors.New("Data source with same name already exists")
ErrDataSourceUpdatingOldVersion = errors.New("Trying to update old version of datasource")
ErrDatasourceIsReadOnly = errors.New("Data source is readonly. Can only be updated from configuration")
ErrDataSourceAccessDenied = errors.New("Data source access denied")
ErrDataSourceNotFound = errors.New("Data source not found")
ErrDataSourceNameExists = errors.New("Data source with the same name already exists")
ErrDataSourceUidExists = errors.New("Data source with the same uid already exists")
ErrDataSourceUpdatingOldVersion = errors.New("Trying to update old version of datasource")
ErrDatasourceIsReadOnly = errors.New("Data source is readonly. Can only be updated from configuration")
ErrDataSourceAccessDenied = errors.New("Data source access denied")
ErrDataSourceFailedGenerateUniqueUid = errors.New("Failed to generate unique datasource id")
)
type DsAccess string
@@ -57,6 +59,7 @@ type DataSource struct {
JsonData *simplejson.Json
SecureJsonData securejsondata.SecureJsonData
ReadOnly bool
Uid string
Created time.Time
Updated time.Time
@@ -144,6 +147,7 @@ type AddDataSourceCommand struct {
IsDefault bool `json:"isDefault"`
JsonData *simplejson.Json `json:"jsonData"`
SecureJsonData map[string]string `json:"secureJsonData"`
Uid string `json:"uid"`
OrgId int64 `json:"-"`
ReadOnly bool `json:"-"`
@@ -168,6 +172,7 @@ type UpdateDataSourceCommand struct {
JsonData *simplejson.Json `json:"jsonData"`
SecureJsonData map[string]string `json:"secureJsonData"`
Version int `json:"version"`
Uid string `json:"uid"`
OrgId int64 `json:"-"`
Id int64 `json:"-"`

View File

@@ -161,7 +161,7 @@ func TestDatasourceAsConfig(t *testing.T) {
So(dsCfg.APIVersion, ShouldEqual, 1)
validateDatasource(dsCfg)
validateDatasourceV1(dsCfg)
validateDeleteDatasources(dsCfg)
dsCount := 0
@@ -231,6 +231,12 @@ func validateDatasource(dsCfg *configs) {
So(ds.SecureJSONData["tlsClientKey"], ShouldEqual, "ZkN4aG1aNkja/gKAB1wlnKFIsy2SRDq4slrM0A==")
}
func validateDatasourceV1(dsCfg *configs) {
validateDatasource(dsCfg)
ds := dsCfg.Datasources[0]
So(ds.UID, ShouldEqual, "test_uid")
}
type fakeRepository struct {
inserted []*models.AddDataSourceCommand
deleted []*models.DeleteDataSourceByNameCommand

View File

@@ -50,13 +50,13 @@ func (dc *DatasourceProvisioner) apply(cfg *configs) error {
}
if err == models.ErrDataSourceNotFound {
dc.log.Info("inserting datasource from configuration ", "name", ds.Name)
dc.log.Info("inserting datasource from configuration ", "name", ds.Name, "uid", ds.UID)
insertCmd := createInsertCommand(ds)
if err := bus.Dispatch(insertCmd); err != nil {
return err
}
} else {
dc.log.Debug("updating datasource from configuration", "name", ds.Name)
dc.log.Debug("updating datasource from configuration", "name", ds.Name, "uid", ds.UID)
updateCmd := createUpdateCommand(ds, cmd.Result.Id)
if err := bus.Dispatch(updateCmd); err != nil {
return err

View File

@@ -24,6 +24,7 @@ datasources:
tlsClientKey: "ZkN4aG1aNkja/gKAB1wlnKFIsy2SRDq4slrM0A=="
editable: true
version: 10
uid: "test_uid"
deleteDatasources:
- name: old-graphite3

View File

@@ -43,6 +43,7 @@ type upsertDataSourceFromConfig struct {
JSONData map[string]interface{}
SecureJSONData map[string]string
Editable bool
UID string
}
type configsV0 struct {
@@ -108,6 +109,7 @@ type upsertDataSourceFromConfigV1 struct {
JSONData values.JSONValue `json:"jsonData" yaml:"jsonData"`
SecureJSONData values.StringMapValue `json:"secureJsonData" yaml:"secureJsonData"`
Editable values.BoolValue `json:"editable" yaml:"editable"`
UID values.StringValue `json:"uid" yaml:"uid"`
}
func (cfg *configsV1) mapToDatasourceFromConfig(apiVersion int64) *configs {
@@ -138,6 +140,7 @@ func (cfg *configsV1) mapToDatasourceFromConfig(apiVersion int64) *configs {
SecureJSONData: ds.SecureJSONData.Value(),
Editable: ds.Editable.Value(),
Version: ds.Version.Value(),
UID: ds.UID.Value(),
})
// Using Raw value for the warnings here so that even if it uses env interpolation and the env var is empty
@@ -234,6 +237,7 @@ func createInsertCommand(ds *upsertDataSourceFromConfig) *models.AddDataSourceCo
JsonData: jsonData,
SecureJsonData: ds.SecureJSONData,
ReadOnly: !ds.Editable,
Uid: ds.UID,
}
}
@@ -247,6 +251,7 @@ func createUpdateCommand(ds *upsertDataSourceFromConfig, id int64) *models.Updat
return &models.UpdateDataSourceCommand{
Id: id,
Uid: ds.UID,
OrgId: ds.OrgID,
Name: ds.Name,
Type: ds.Type,

View File

@@ -1,6 +1,8 @@
package sqlstore
import (
"github.com/grafana/grafana/pkg/util/errutil"
"strings"
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
@@ -101,6 +103,14 @@ func AddDataSource(cmd *models.AddDataSourceCommand) error {
cmd.JsonData = simplejson.New()
}
if cmd.Uid == "" {
uid, err := generateNewDatasourceUid(sess, cmd.OrgId)
if err != nil {
return errutil.Wrapf(err, "Failed to generate UID for datasource %q", cmd.Name)
}
cmd.Uid = uid
}
ds := &models.DataSource{
OrgId: cmd.OrgId,
Name: cmd.Name,
@@ -121,9 +131,13 @@ func AddDataSource(cmd *models.AddDataSourceCommand) error {
Updated: time.Now(),
Version: 1,
ReadOnly: cmd.ReadOnly,
Uid: cmd.Uid,
}
if _, err := sess.Insert(ds); err != nil {
if dialect.IsUniqueConstraintViolation(err) && strings.Contains(strings.ToLower(dialect.ErrorMessage(err)), "uid") {
return models.ErrDataSourceUidExists
}
return err
}
if err := updateIsDefaultFlag(ds, sess); err != nil {
@@ -172,6 +186,7 @@ func UpdateDataSource(cmd *models.UpdateDataSourceCommand) error {
Updated: time.Now(),
ReadOnly: cmd.ReadOnly,
Version: cmd.Version + 1,
Uid: cmd.Uid,
}
sess.UseBool("is_default")
@@ -209,3 +224,20 @@ func UpdateDataSource(cmd *models.UpdateDataSourceCommand) error {
return err
})
}
func generateNewDatasourceUid(sess *DBSession, orgId int64) (string, error) {
for i := 0; i < 3; i++ {
uid := generateNewUid()
exists, err := sess.Where("org_id=? AND uid=?", orgId, uid).Get(&models.DataSource{})
if err != nil {
return "", err
}
if !exists {
return uid, nil
}
}
return "", models.ErrDataSourceFailedGenerateUniqueUid
}

View File

@@ -3,173 +3,213 @@ package sqlstore
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
"github.com/grafana/grafana/pkg/models"
"github.com/stretchr/testify/require"
)
type Test struct {
Id int64
Name string
}
func TestDataAccess(t *testing.T) {
Convey("Testing DB", t, func() {
InitTestDB(t)
Convey("Can add datasource", func() {
defaultAddDatasourceCommand := models.AddDataSourceCommand{
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_DIRECT,
Url: "http://test",
}
defaultUpdateDatasourceCommand := models.UpdateDataSourceCommand{
OrgId: 10,
Name: "nisse_updated",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_DIRECT,
Url: "http://test",
}
initDatasource := func() *models.DataSource {
cmd := defaultAddDatasourceCommand
err := AddDataSource(&cmd)
require.NoError(t, err)
query := models.GetDataSourcesQuery{OrgId: 10}
err = GetDataSources(&query)
require.NoError(t, err)
require.Equal(t, 1, len(query.Result))
return query.Result[0]
}
t.Run("AddDataSource", func(t *testing.T) {
t.Run("Can add datasource", func(t *testing.T) {
InitTestDB(t)
err := AddDataSource(&models.AddDataSourceCommand{
OrgId: 10,
Name: "laban",
Type: models.DS_INFLUXDB,
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_DIRECT,
Url: "http://test",
Database: "site",
ReadOnly: true,
})
So(err, ShouldBeNil)
require.NoError(t, err)
query := models.GetDataSourcesQuery{OrgId: 10}
err = GetDataSources(&query)
So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 1)
require.NoError(t, err)
require.Equal(t, 1, len(query.Result))
ds := query.Result[0]
So(ds.OrgId, ShouldEqual, 10)
So(ds.Database, ShouldEqual, "site")
So(ds.ReadOnly, ShouldBeTrue)
require.EqualValues(t, 10, ds.OrgId)
require.Equal(t, "site", ds.Database)
require.True(t, ds.ReadOnly)
})
Convey("Given a datasource", func() {
err := AddDataSource(&models.AddDataSourceCommand{
t.Run("generates uid if not specified", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
require.NotEmpty(t, ds.Uid)
})
t.Run("fails to insert ds with same uid", func(t *testing.T) {
InitTestDB(t)
cmd1 := defaultAddDatasourceCommand
cmd2 := defaultAddDatasourceCommand
cmd1.Uid = "test"
cmd2.Uid = "test"
err := AddDataSource(&cmd1)
require.NoError(t, err)
err = AddDataSource(&cmd2)
require.Error(t, err)
require.IsType(t, models.ErrDataSourceUidExists, err)
})
})
t.Run("UpdateDataSource", func(t *testing.T) {
t.Run("updates datasource with version", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
cmd := defaultUpdateDatasourceCommand
cmd.Id = ds.Id
cmd.Version = ds.Version
err := UpdateDataSource(&cmd)
require.NoError(t, err)
})
t.Run("does not overwrite Uid if not specified", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
require.NotEmpty(t, ds.Uid)
cmd := defaultUpdateDatasourceCommand
cmd.Id = ds.Id
err := UpdateDataSource(&cmd)
require.NoError(t, err)
query := models.GetDataSourceByIdQuery{Id: ds.Id}
err = GetDataSourceById(&query)
require.NoError(t, err)
require.Equal(t, ds.Uid, query.Result.Uid)
})
t.Run("prevents update if version changed", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
cmd := models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
Version: ds.Version,
}
// Make a copy as UpdateDataSource modifies it
cmd2 := cmd
err := UpdateDataSource(&cmd)
require.NoError(t, err)
err = UpdateDataSource(&cmd2)
require.Error(t, err)
})
t.Run("updates ds without version specified", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
cmd := &models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_DIRECT,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
})
So(err, ShouldBeNil)
}
err := UpdateDataSource(cmd)
require.NoError(t, err)
})
t.Run("updates ds without higher version", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
cmd := &models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
Version: 90000,
}
err := UpdateDataSource(cmd)
require.NoError(t, err)
})
})
t.Run("DeleteDataSourceById", func(t *testing.T) {
t.Run("can delete datasource", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
err := DeleteDataSourceById(&models.DeleteDataSourceByIdCommand{Id: ds.Id, OrgId: ds.OrgId})
require.NoError(t, err)
query := models.GetDataSourcesQuery{OrgId: 10}
err = GetDataSources(&query)
So(err, ShouldBeNil)
require.NoError(t, err)
ds := query.Result[0]
require.Equal(t, 0, len(query.Result))
})
Convey(" updated ", func() {
cmd := &models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
Version: ds.Version,
}
t.Run("Can not delete datasource with wrong orgId", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
Convey("with same version as source", func() {
err := UpdateDataSource(cmd)
So(err, ShouldBeNil)
})
err := DeleteDataSourceById(&models.DeleteDataSourceByIdCommand{Id: ds.Id, OrgId: 123123})
require.NoError(t, err)
query := models.GetDataSourcesQuery{OrgId: 10}
err = GetDataSources(&query)
require.NoError(t, err)
Convey("when someone else updated between read and update", func() {
query := models.GetDataSourcesQuery{OrgId: 10}
err = GetDataSources(&query)
So(err, ShouldBeNil)
ds := query.Result[0]
intendedUpdate := &models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
Version: ds.Version,
}
updateFromOtherUser := &models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
Version: ds.Version,
}
err := UpdateDataSource(updateFromOtherUser)
So(err, ShouldBeNil)
err = UpdateDataSource(intendedUpdate)
So(err, ShouldNotBeNil)
})
Convey("updating datasource without version", func() {
cmd := &models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
}
Convey("should not raise errors", func() {
err := UpdateDataSource(cmd)
So(err, ShouldBeNil)
})
})
Convey("updating datasource without higher version", func() {
cmd := &models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
Version: 90000,
}
Convey("should not raise errors", func() {
err := UpdateDataSource(cmd)
So(err, ShouldBeNil)
})
})
})
Convey("Can delete datasource by id", func() {
err := DeleteDataSourceById(&models.DeleteDataSourceByIdCommand{Id: ds.Id, OrgId: ds.OrgId})
So(err, ShouldBeNil)
err = GetDataSources(&query)
So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 0)
})
Convey("Can delete datasource by name", func() {
err := DeleteDataSourceByName(&models.DeleteDataSourceByNameCommand{Name: ds.Name, OrgId: ds.OrgId})
So(err, ShouldBeNil)
err = GetDataSources(&query)
So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 0)
})
Convey("Can not delete datasource with wrong orgId", func() {
err := DeleteDataSourceById(&models.DeleteDataSourceByIdCommand{Id: ds.Id, OrgId: 123123})
So(err, ShouldBeNil)
err = GetDataSources(&query)
So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 1)
})
require.Equal(t, 1, len(query.Result))
})
})
t.Run("DeleteDataSourceByName", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
query := models.GetDataSourcesQuery{OrgId: 10}
err := DeleteDataSourceByName(&models.DeleteDataSourceByNameCommand{Name: ds.Name, OrgId: ds.OrgId})
require.NoError(t, err)
err = GetDataSources(&query)
require.NoError(t, err)
require.Equal(t, 0, len(query.Result))
})
}

View File

@@ -133,4 +133,22 @@ func addDataSourceMigration(mg *Migrator) {
const setEmptyJSONWhereNullJSON = `UPDATE data_source SET json_data = '{}' WHERE json_data is null`
mg.AddMigration("Update json_data with nulls", NewRawSqlMigration(setEmptyJSONWhereNullJSON))
// add column uid for linking
mg.AddMigration("Add uid column", NewAddColumnMigration(tableV2, &Column{
Name: "uid", Type: DB_NVarchar, Length: 40, Nullable: false, Default: "0",
}))
// Initialize as id as that is unique already
mg.AddMigration(
"Update uid value",
NewRawSqlMigration("").
Sqlite("UPDATE data_source SET uid=printf('%09d',id);").
Postgres("UPDATE data_source SET uid=lpad('' || id::text,9,'0');").
Mysql("UPDATE data_source SET uid=lpad(id,9,'0');"),
)
mg.AddMigration("Add unique index datasource_org_id_uid", NewAddIndexMigration(tableV2, &Index{
Cols: []string{"org_id", "uid"}, Type: UniqueIndex,
}))
}

View File

@@ -48,6 +48,7 @@ type Dialect interface {
NoOpSql() string
IsUniqueConstraintViolation(err error) bool
ErrorMessage(err error) string
IsDeadlock(err error) bool
}

View File

@@ -148,6 +148,13 @@ func (db *Mysql) IsUniqueConstraintViolation(err error) bool {
return db.isThisError(err, mysqlerr.ER_DUP_ENTRY)
}
func (db *Mysql) ErrorMessage(err error) string {
if driverErr, ok := err.(*mysql.MySQLError); ok {
return driverErr.Message
}
return ""
}
func (db *Mysql) IsDeadlock(err error) bool {
return db.isThisError(err, mysqlerr.ER_LOCK_DEADLOCK)
}

View File

@@ -2,11 +2,11 @@ package migrator
import (
"fmt"
"github.com/lib/pq"
"strconv"
"strings"
"github.com/grafana/grafana/pkg/util/errutil"
"github.com/lib/pq"
"xorm.io/xorm"
)
@@ -149,6 +149,13 @@ func (db *Postgres) isThisError(err error, errcode string) bool {
return false
}
func (db *Postgres) ErrorMessage(err error) string {
if driverErr, ok := err.(*pq.Error); ok {
return driverErr.Message
}
return ""
}
func (db *Postgres) IsUniqueConstraintViolation(err error) bool {
return db.isThisError(err, "23505")
}

View File

@@ -2,7 +2,6 @@ package migrator
import (
"fmt"
"github.com/mattn/go-sqlite3"
"xorm.io/xorm"
)
@@ -95,6 +94,13 @@ func (db *Sqlite3) isThisError(err error, errcode int) bool {
return false
}
func (db *Sqlite3) ErrorMessage(err error) string {
if driverErr, ok := err.(sqlite3.Error); ok {
return driverErr.Error()
}
return ""
}
func (db *Sqlite3) IsUniqueConstraintViolation(err error) bool {
return db.isThisError(err, int(sqlite3.ErrConstraintUnique))
}