From e4741ce8d60b862b5c10883f5d84b5c5e258b917 Mon Sep 17 00:00:00 2001 From: ying-jeanne <74549700+ying-jeanne@users.noreply.github.com> Date: Fri, 16 Sep 2022 05:20:26 -0500 Subject: [PATCH] remove datasource from sqlstore (#55288) --- .../accesscontrol/filter_bench_test.go | 8 +- pkg/services/accesscontrol/filter_test.go | 5 +- .../resourcepermissions/store_bench_test.go | 6 +- pkg/services/export/export_ds.go | 2 +- pkg/services/export/git_export_job.go | 5 +- pkg/services/export/service.go | 7 +- pkg/services/publicdashboards/api/api_test.go | 5 +- pkg/services/sqlstore/datasource.go | 324 ----------- pkg/services/sqlstore/datasource_test.go | 506 ------------------ 9 files changed, 26 insertions(+), 842 deletions(-) delete mode 100644 pkg/services/sqlstore/datasource.go delete mode 100644 pkg/services/sqlstore/datasource_test.go diff --git a/pkg/services/accesscontrol/filter_bench_test.go b/pkg/services/accesscontrol/filter_bench_test.go index 1ca58448cdc..7cd5b788d63 100644 --- a/pkg/services/accesscontrol/filter_bench_test.go +++ b/pkg/services/accesscontrol/filter_bench_test.go @@ -8,8 +8,10 @@ import ( "github.com/stretchr/testify/require" + "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/datasources" + dsService "github.com/grafana/grafana/pkg/services/datasources/service" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/user" ) @@ -51,8 +53,8 @@ func benchmarkFilter(b *testing.B, numDs, numPermissions int) { func setupFilterBenchmark(b *testing.B, numDs, numPermissions int) (*sqlstore.SQLStore, []accesscontrol.Permission) { b.Helper() - store := sqlstore.InitTestDB(b) - + sqlStore := sqlstore.InitTestDB(b) + store := dsService.CreateStore(sqlStore, log.New("accesscontrol.test")) for i := 1; i <= numDs; i++ { err := store.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{ Name: fmt.Sprintf("ds:%d", i), @@ -73,5 +75,5 @@ func setupFilterBenchmark(b *testing.B, numDs, numPermissions int) (*sqlstore.SQ }) } - return store, permissions + return sqlStore, permissions } diff --git a/pkg/services/accesscontrol/filter_test.go b/pkg/services/accesscontrol/filter_test.go index 897406a47ba..cb343a8ae5d 100644 --- a/pkg/services/accesscontrol/filter_test.go +++ b/pkg/services/accesscontrol/filter_test.go @@ -8,8 +8,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/datasources" + dsService "github.com/grafana/grafana/pkg/services/datasources/service" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/user" ) @@ -171,7 +173,8 @@ func TestFilter_Datasources(t *testing.T) { // seed 10 data sources for i := 1; i <= 10; i++ { - err := store.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{Name: fmt.Sprintf("ds:%d", i), Uid: fmt.Sprintf("uid%d", i)}) + dsStore := dsService.CreateStore(store, log.New("accesscontrol.test")) + err := dsStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{Name: fmt.Sprintf("ds:%d", i), Uid: fmt.Sprintf("uid%d", i)}) require.NoError(t, err) } diff --git a/pkg/services/accesscontrol/resourcepermissions/store_bench_test.go b/pkg/services/accesscontrol/resourcepermissions/store_bench_test.go index cd35e08de19..9b9e0cf2edf 100644 --- a/pkg/services/accesscontrol/resourcepermissions/store_bench_test.go +++ b/pkg/services/accesscontrol/resourcepermissions/store_bench_test.go @@ -10,8 +10,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/datasources" + datasourcesService "github.com/grafana/grafana/pkg/services/datasources/service" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/user" ) @@ -82,8 +84,8 @@ func GenerateDatasourcePermissions(b *testing.B, db *sqlstore.SQLStore, ac *stor Access: datasources.DS_ACCESS_DIRECT, Url: "http://test", } - - _ = db.AddDataSource(context.Background(), addDSCommand) + dsStore := datasourcesService.CreateStore(db, log.New("publicdashboards.test")) + _ = dsStore.AddDataSource(context.Background(), addDSCommand) dataSources = append(dataSources, addDSCommand.Result.Id) } diff --git a/pkg/services/export/export_ds.go b/pkg/services/export/export_ds.go index c7b9e5ca49c..ac8e76cf22e 100644 --- a/pkg/services/export/export_ds.go +++ b/pkg/services/export/export_ds.go @@ -12,7 +12,7 @@ func exportDataSources(helper *commitHelper, job *gitExportJob) error { cmd := &datasources.GetDataSourcesQuery{ OrgId: helper.orgID, } - err := job.sql.GetDataSources(helper.ctx, cmd) + err := job.datasourceService.GetDataSources(helper.ctx, cmd) if err != nil { return nil } diff --git a/pkg/services/export/git_export_job.go b/pkg/services/export/git_export_job.go index fa8c36d7339..f4bff6c0092 100644 --- a/pkg/services/export/git_export_job.go +++ b/pkg/services/export/git_export_job.go @@ -14,6 +14,7 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/dashboardsnapshots" + "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/playlist" "github.com/grafana/grafana/pkg/services/sqlstore" ) @@ -24,6 +25,7 @@ type gitExportJob struct { logger log.Logger sql *sqlstore.SQLStore dashboardsnapshotsService dashboardsnapshots.Service + datasourceService datasources.DataSourceService playlistService playlist.Service rootDir string @@ -36,13 +38,14 @@ type gitExportJob struct { func startGitExportJob(cfg ExportConfig, sql *sqlstore.SQLStore, dashboardsnapshotsService dashboardsnapshots.Service, rootDir string, orgID int64, - broadcaster statusBroadcaster, playlistService playlist.Service) (Job, error) { + broadcaster statusBroadcaster, playlistService playlist.Service, datasourceService datasources.DataSourceService) (Job, error) { job := &gitExportJob{ logger: log.New("git_export_job"), cfg: cfg, sql: sql, dashboardsnapshotsService: dashboardsnapshotsService, playlistService: playlistService, + datasourceService: datasourceService, rootDir: rootDir, broadcaster: broadcaster, status: ExportStatus{ diff --git a/pkg/services/export/service.go b/pkg/services/export/service.go index 28c83bf5d64..69945f6df2f 100644 --- a/pkg/services/export/service.go +++ b/pkg/services/export/service.go @@ -13,6 +13,7 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/dashboardsnapshots" + "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/live" "github.com/grafana/grafana/pkg/services/playlist" @@ -152,13 +153,14 @@ type StandardExport struct { sql *sqlstore.SQLStore dashboardsnapshotsService dashboardsnapshots.Service playlistService playlist.Service + datasourceService datasources.DataSourceService // updated with mutex exportJob Job } func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, gl *live.GrafanaLive, cfg *setting.Cfg, - dashboardsnapshotsService dashboardsnapshots.Service, playlistService playlist.Service) ExportService { + dashboardsnapshotsService dashboardsnapshots.Service, playlistService playlist.Service, datasourceService datasources.DataSourceService) ExportService { if !features.IsEnabled(featuremgmt.FlagExport) { return &StubExport{} } @@ -169,6 +171,7 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, logger: log.New("export_service"), dashboardsnapshotsService: dashboardsnapshotsService, playlistService: playlistService, + datasourceService: datasourceService, exportJob: &stoppedJob{}, dataDir: cfg.DataPath, } @@ -225,7 +228,7 @@ func (ex *StandardExport) HandleRequestExport(c *models.ReqContext) response.Res if err := os.MkdirAll(dir, os.ModePerm); err != nil { return response.Error(http.StatusBadRequest, "Error creating export folder", nil) } - job, err = startGitExportJob(cfg, ex.sql, ex.dashboardsnapshotsService, dir, c.OrgID, broadcast, ex.playlistService) + job, err = startGitExportJob(cfg, ex.sql, ex.dashboardsnapshotsService, dir, c.OrgID, broadcast, ex.playlistService, ex.datasourceService) default: return response.Error(http.StatusBadRequest, "Unsupported job format", nil) } diff --git a/pkg/services/publicdashboards/api/api_test.go b/pkg/services/publicdashboards/api/api_test.go index 54438d60834..494bedb457b 100644 --- a/pkg/services/publicdashboards/api/api_test.go +++ b/pkg/services/publicdashboards/api/api_test.go @@ -20,6 +20,7 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/localcache" + "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/dashboards" dashboardStore "github.com/grafana/grafana/pkg/services/dashboards/database" @@ -487,8 +488,8 @@ func TestIntegrationUnauthenticatedUserCanGetPubdashPanelQueryData(t *testing.T) cacheService := datasourcesService.ProvideCacheService(localcache.ProvideService(), db) qds := buildQueryDataService(t, cacheService, nil, db) - - _ = db.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{ + dsStore := datasourcesService.CreateStore(db, log.New("publicdashboards.test")) + _ = dsStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{ Uid: "ds1", OrgId: 1, Name: "laban", diff --git a/pkg/services/sqlstore/datasource.go b/pkg/services/sqlstore/datasource.go deleted file mode 100644 index fab74f05f8e..00000000000 --- a/pkg/services/sqlstore/datasource.go +++ /dev/null @@ -1,324 +0,0 @@ -package sqlstore - -import ( - "context" - "errors" - "fmt" - "strings" - "time" - - "xorm.io/xorm" - - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/events" - "github.com/grafana/grafana/pkg/infra/metrics" - ac "github.com/grafana/grafana/pkg/services/accesscontrol" - "github.com/grafana/grafana/pkg/services/datasources" - "github.com/grafana/grafana/pkg/util" -) - -// GetDataSource adds a datasource to the query model by querying by org_id as well as -// either uid (preferred), id, or name and is added to the bus. -func (ss *SQLStore) GetDataSource(ctx context.Context, query *datasources.GetDataSourceQuery) error { - metrics.MDBDataSourceQueryByID.Inc() - - return ss.WithDbSession(ctx, func(sess *DBSession) error { - return ss.getDataSource(ctx, query, sess) - }) -} - -func (ss *SQLStore) getDataSource(ctx context.Context, query *datasources.GetDataSourceQuery, sess *DBSession) error { - if query.OrgId == 0 || (query.Id == 0 && len(query.Name) == 0 && len(query.Uid) == 0) { - return datasources.ErrDataSourceIdentifierNotSet - } - - datasource := &datasources.DataSource{Name: query.Name, OrgId: query.OrgId, Id: query.Id, Uid: query.Uid} - has, err := sess.Get(datasource) - - if err != nil { - sqlog.Error("Failed getting data source", "err", err, "uid", query.Uid, "id", query.Id, "name", query.Name, "orgId", query.OrgId) - return err - } else if !has { - return datasources.ErrDataSourceNotFound - } - - query.Result = datasource - - return nil -} - -func (ss *SQLStore) GetDataSources(ctx context.Context, query *datasources.GetDataSourcesQuery) error { - var sess *xorm.Session - return ss.WithDbSession(ctx, func(dbSess *DBSession) error { - if query.DataSourceLimit <= 0 { - sess = dbSess.Where("org_id=?", query.OrgId).Asc("name") - } else { - sess = dbSess.Limit(query.DataSourceLimit, 0).Where("org_id=?", query.OrgId).Asc("name") - } - - query.Result = make([]*datasources.DataSource, 0) - return sess.Find(&query.Result) - }) -} - -func (ss *SQLStore) GetAllDataSources(ctx context.Context, query *datasources.GetAllDataSourcesQuery) error { - return ss.WithDbSession(ctx, func(sess *DBSession) error { - query.Result = make([]*datasources.DataSource, 0) - return sess.Asc("name").Find(&query.Result) - }) -} - -// GetDataSourcesByType returns all datasources for a given type or an error if the specified type is an empty string -func (ss *SQLStore) GetDataSourcesByType(ctx context.Context, query *datasources.GetDataSourcesByTypeQuery) error { - if query.Type == "" { - return fmt.Errorf("datasource type cannot be empty") - } - - query.Result = make([]*datasources.DataSource, 0) - return ss.WithDbSession(ctx, func(sess *DBSession) error { - if query.OrgId > 0 { - return sess.Where("type=? AND org_id=?", query.Type, query.OrgId).Asc("id").Find(&query.Result) - } - return sess.Where("type=?", query.Type).Asc("id").Find(&query.Result) - }) -} - -// GetDefaultDataSource is used to get the default datasource of organization -func (ss *SQLStore) GetDefaultDataSource(ctx context.Context, query *datasources.GetDefaultDataSourceQuery) error { - datasource := datasources.DataSource{} - return ss.WithDbSession(ctx, func(sess *DBSession) error { - exists, err := sess.Where("org_id=? AND is_default=?", query.OrgId, true).Get(&datasource) - - if !exists { - return datasources.ErrDataSourceNotFound - } - - query.Result = &datasource - return err - }) -} - -// DeleteDataSource removes a datasource by org_id as well as either uid (preferred), id, or name -// and is added to the bus. It also removes permissions related to the datasource. -func (ss *SQLStore) DeleteDataSource(ctx context.Context, cmd *datasources.DeleteDataSourceCommand) error { - return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { - dsQuery := &datasources.GetDataSourceQuery{Id: cmd.ID, Uid: cmd.UID, Name: cmd.Name, OrgId: cmd.OrgID} - errGettingDS := ss.getDataSource(ctx, dsQuery, sess) - - if errGettingDS != nil && !errors.Is(errGettingDS, datasources.ErrDataSourceNotFound) { - return errGettingDS - } - - ds := dsQuery.Result - if ds != nil { - // Delete the data source - result, err := sess.Exec("DELETE FROM data_source WHERE org_id=? AND id=?", ds.OrgId, ds.Id) - if err != nil { - return err - } - - cmd.DeletedDatasourcesCount, _ = result.RowsAffected() - - // Remove associated AccessControl permissions - if _, errDeletingPerms := sess.Exec("DELETE FROM permission WHERE scope=?", - ac.Scope("datasources", "id", fmt.Sprint(dsQuery.Result.Id))); errDeletingPerms != nil { - return errDeletingPerms - } - } - - if cmd.UpdateSecretFn != nil { - if err := cmd.UpdateSecretFn(); err != nil { - sqlog.Error("Failed to update datasource secrets -- rolling back update", "UID", cmd.UID, "name", cmd.Name, "orgId", cmd.OrgID) - return err - } - } - - // Publish data source deletion event - if cmd.DeletedDatasourcesCount > 0 { - sess.publishAfterCommit(&events.DataSourceDeleted{ - Timestamp: time.Now(), - Name: ds.Name, - ID: ds.Id, - UID: ds.Uid, - OrgID: ds.OrgId, - }) - } - - return nil - }) -} - -func (ss *SQLStore) AddDataSource(ctx context.Context, cmd *datasources.AddDataSourceCommand) error { - return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { - existing := datasources.DataSource{OrgId: cmd.OrgId, Name: cmd.Name} - has, _ := sess.Get(&existing) - - if has { - return datasources.ErrDataSourceNameExists - } - - if cmd.JsonData == nil { - cmd.JsonData = simplejson.New() - } - - if cmd.Uid == "" { - uid, err := generateNewDatasourceUid(sess, cmd.OrgId) - if err != nil { - return fmt.Errorf("failed to generate UID for datasource %q: %w", cmd.Name, err) - } - cmd.Uid = uid - } - - ds := &datasources.DataSource{ - OrgId: cmd.OrgId, - Name: cmd.Name, - Type: cmd.Type, - Access: cmd.Access, - Url: cmd.Url, - User: cmd.User, - Database: cmd.Database, - IsDefault: cmd.IsDefault, - BasicAuth: cmd.BasicAuth, - BasicAuthUser: cmd.BasicAuthUser, - WithCredentials: cmd.WithCredentials, - JsonData: cmd.JsonData, - SecureJsonData: cmd.EncryptedSecureJsonData, - Created: time.Now(), - Updated: time.Now(), - Version: 1, - ReadOnly: cmd.ReadOnly, - Uid: cmd.Uid, - } - - if _, err := sess.Insert(ds); err != nil { - if dialect.IsUniqueConstraintViolation(err) && strings.Contains(strings.ToLower(dialect.ErrorMessage(err)), "uid") { - return datasources.ErrDataSourceUidExists - } - return err - } - if err := updateIsDefaultFlag(ds, sess); err != nil { - return err - } - - if cmd.UpdateSecretFn != nil { - if err := cmd.UpdateSecretFn(); err != nil { - sqlog.Error("Failed to update datasource secrets -- rolling back update", "name", cmd.Name, "type", cmd.Type, "orgId", cmd.OrgId) - return err - } - } - - cmd.Result = ds - - sess.publishAfterCommit(&events.DataSourceCreated{ - Timestamp: time.Now(), - Name: cmd.Name, - ID: ds.Id, - UID: cmd.Uid, - OrgID: cmd.OrgId, - }) - return nil - }) -} - -func updateIsDefaultFlag(ds *datasources.DataSource, sess *DBSession) error { - // Handle is default flag - if ds.IsDefault { - rawSQL := "UPDATE data_source SET is_default=? WHERE org_id=? AND id <> ?" - if _, err := sess.Exec(rawSQL, false, ds.OrgId, ds.Id); err != nil { - return err - } - } - return nil -} - -func (ss *SQLStore) UpdateDataSource(ctx context.Context, cmd *datasources.UpdateDataSourceCommand) error { - return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { - if cmd.JsonData == nil { - cmd.JsonData = simplejson.New() - } - - ds := &datasources.DataSource{ - Id: cmd.Id, - OrgId: cmd.OrgId, - Name: cmd.Name, - Type: cmd.Type, - Access: cmd.Access, - Url: cmd.Url, - User: cmd.User, - Database: cmd.Database, - IsDefault: cmd.IsDefault, - BasicAuth: cmd.BasicAuth, - BasicAuthUser: cmd.BasicAuthUser, - WithCredentials: cmd.WithCredentials, - JsonData: cmd.JsonData, - SecureJsonData: cmd.EncryptedSecureJsonData, - Updated: time.Now(), - ReadOnly: cmd.ReadOnly, - Version: cmd.Version + 1, - Uid: cmd.Uid, - } - - sess.UseBool("is_default") - sess.UseBool("basic_auth") - sess.UseBool("with_credentials") - sess.UseBool("read_only") - // Make sure password are zeroed out if empty. We do this as we want to migrate passwords from - // plain text fields to SecureJsonData. - sess.MustCols("password") - sess.MustCols("basic_auth_password") - sess.MustCols("user") - // Make sure secure json data is zeroed out if empty. We do this as we want to migrate secrets from - // secure json data to the unified secrets table. - sess.MustCols("secure_json_data") - - var updateSession *xorm.Session - if cmd.Version != 0 { - // the reason we allow cmd.version > db.version is make it possible for people to force - // updates to datasources using the datasource.yaml file without knowing exactly what version - // a datasource have in the db. - updateSession = sess.Where("id=? and org_id=? and version < ?", ds.Id, ds.OrgId, ds.Version) - } else { - updateSession = sess.Where("id=? and org_id=?", ds.Id, ds.OrgId) - } - - affected, err := updateSession.Update(ds) - if err != nil { - return err - } - - if affected == 0 { - return datasources.ErrDataSourceUpdatingOldVersion - } - - err = updateIsDefaultFlag(ds, sess) - - if cmd.UpdateSecretFn != nil { - if err := cmd.UpdateSecretFn(); err != nil { - sqlog.Error("Failed to update datasource secrets -- rolling back update", "UID", cmd.Uid, "name", cmd.Name, "type", cmd.Type, "orgId", cmd.OrgId) - return err - } - } - - cmd.Result = ds - return err - }) -} - -func generateNewDatasourceUid(sess *DBSession, orgId int64) (string, error) { - for i := 0; i < 3; i++ { - uid := generateNewUid() - - exists, err := sess.Where("org_id=? AND uid=?", orgId, uid).Get(&datasources.DataSource{}) - if err != nil { - return "", err - } - - if !exists { - return uid, nil - } - } - - return "", datasources.ErrDataSourceFailedGenerateUniqueUid -} - -var generateNewUid func() string = util.GenerateShortUID diff --git a/pkg/services/sqlstore/datasource_test.go b/pkg/services/sqlstore/datasource_test.go deleted file mode 100644 index 34169f799a9..00000000000 --- a/pkg/services/sqlstore/datasource_test.go +++ /dev/null @@ -1,506 +0,0 @@ -package sqlstore - -import ( - "context" - "errors" - "fmt" - "strconv" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/events" - ac "github.com/grafana/grafana/pkg/services/accesscontrol" - "github.com/grafana/grafana/pkg/services/datasources" -) - -func TestIntegrationDataAccess(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - defaultAddDatasourceCommand := datasources.AddDataSourceCommand{ - OrgId: 10, - Name: "nisse", - Type: datasources.DS_GRAPHITE, - Access: datasources.DS_ACCESS_DIRECT, - Url: "http://test", - } - - defaultUpdateDatasourceCommand := datasources.UpdateDataSourceCommand{ - OrgId: 10, - Name: "nisse_updated", - Type: datasources.DS_GRAPHITE, - Access: datasources.DS_ACCESS_DIRECT, - Url: "http://test", - } - - initDatasource := func(sqlStore *SQLStore) *datasources.DataSource { - cmd := defaultAddDatasourceCommand - err := sqlStore.AddDataSource(context.Background(), &cmd) - require.NoError(t, err) - - query := datasources.GetDataSourcesQuery{OrgId: 10} - err = sqlStore.GetDataSources(context.Background(), &query) - require.NoError(t, err) - require.Equal(t, 1, len(query.Result)) - - return query.Result[0] - } - - t.Run("AddDataSource", func(t *testing.T) { - t.Run("Can add datasource", func(t *testing.T) { - sqlStore := InitTestDB(t) - - err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{ - OrgId: 10, - Name: "laban", - Type: datasources.DS_GRAPHITE, - Access: datasources.DS_ACCESS_DIRECT, - Url: "http://test", - Database: "site", - ReadOnly: true, - }) - require.NoError(t, err) - - query := datasources.GetDataSourcesQuery{OrgId: 10} - err = sqlStore.GetDataSources(context.Background(), &query) - require.NoError(t, err) - - require.Equal(t, 1, len(query.Result)) - ds := query.Result[0] - - require.EqualValues(t, 10, ds.OrgId) - require.Equal(t, "site", ds.Database) - require.True(t, ds.ReadOnly) - }) - - t.Run("generates uid if not specified", func(t *testing.T) { - sqlStore := InitTestDB(t) - ds := initDatasource(sqlStore) - require.NotEmpty(t, ds.Uid) - }) - - t.Run("fails to insert ds with same uid", func(t *testing.T) { - sqlStore := InitTestDB(t) - cmd1 := defaultAddDatasourceCommand - cmd2 := defaultAddDatasourceCommand - cmd1.Uid = "test" - cmd2.Uid = "test" - err := sqlStore.AddDataSource(context.Background(), &cmd1) - require.NoError(t, err) - err = sqlStore.AddDataSource(context.Background(), &cmd2) - require.Error(t, err) - require.IsType(t, datasources.ErrDataSourceUidExists, err) - }) - - t.Run("fires an event when the datasource is added", func(t *testing.T) { - sqlStore := InitTestDB(t) - - var created *events.DataSourceCreated - sqlStore.bus.AddEventListener(func(ctx context.Context, e *events.DataSourceCreated) error { - created = e - return nil - }) - - err := sqlStore.AddDataSource(context.Background(), &defaultAddDatasourceCommand) - require.NoError(t, err) - - require.Eventually(t, func() bool { - return assert.NotNil(t, created) - }, time.Second, time.Millisecond) - - query := datasources.GetDataSourcesQuery{OrgId: 10} - err = sqlStore.GetDataSources(context.Background(), &query) - require.NoError(t, err) - require.Equal(t, 1, len(query.Result)) - - require.Equal(t, query.Result[0].Id, created.ID) - require.Equal(t, query.Result[0].Uid, created.UID) - require.Equal(t, int64(10), created.OrgID) - require.Equal(t, "nisse", created.Name) - }) - }) - - t.Run("UpdateDataSource", func(t *testing.T) { - t.Run("updates datasource with version", func(t *testing.T) { - sqlStore := InitTestDB(t) - ds := initDatasource(sqlStore) - cmd := defaultUpdateDatasourceCommand - cmd.Id = ds.Id - cmd.Version = ds.Version - err := sqlStore.UpdateDataSource(context.Background(), &cmd) - require.NoError(t, err) - }) - - t.Run("does not overwrite Uid if not specified", func(t *testing.T) { - sqlStore := InitTestDB(t) - ds := initDatasource(sqlStore) - require.NotEmpty(t, ds.Uid) - - cmd := defaultUpdateDatasourceCommand - cmd.Id = ds.Id - err := sqlStore.UpdateDataSource(context.Background(), &cmd) - require.NoError(t, err) - - query := datasources.GetDataSourceQuery{Id: ds.Id, OrgId: 10} - err = sqlStore.GetDataSource(context.Background(), &query) - require.NoError(t, err) - require.Equal(t, ds.Uid, query.Result.Uid) - }) - - t.Run("prevents update if version changed", func(t *testing.T) { - sqlStore := InitTestDB(t) - ds := initDatasource(sqlStore) - - cmd := datasources.UpdateDataSourceCommand{ - Id: ds.Id, - OrgId: 10, - Name: "nisse", - Type: datasources.DS_GRAPHITE, - Access: datasources.DS_ACCESS_PROXY, - Url: "http://test", - Version: ds.Version, - } - // Make a copy as UpdateDataSource modifies it - cmd2 := cmd - - err := sqlStore.UpdateDataSource(context.Background(), &cmd) - require.NoError(t, err) - - err = sqlStore.UpdateDataSource(context.Background(), &cmd2) - require.Error(t, err) - }) - - t.Run("updates ds without version specified", func(t *testing.T) { - sqlStore := InitTestDB(t) - ds := initDatasource(sqlStore) - - cmd := &datasources.UpdateDataSourceCommand{ - Id: ds.Id, - OrgId: 10, - Name: "nisse", - Type: datasources.DS_GRAPHITE, - Access: datasources.DS_ACCESS_PROXY, - Url: "http://test", - } - - err := sqlStore.UpdateDataSource(context.Background(), cmd) - require.NoError(t, err) - }) - - t.Run("updates ds without higher version", func(t *testing.T) { - sqlStore := InitTestDB(t) - ds := initDatasource(sqlStore) - - cmd := &datasources.UpdateDataSourceCommand{ - Id: ds.Id, - OrgId: 10, - Name: "nisse", - Type: datasources.DS_GRAPHITE, - Access: datasources.DS_ACCESS_PROXY, - Url: "http://test", - Version: 90000, - } - - err := sqlStore.UpdateDataSource(context.Background(), cmd) - require.NoError(t, err) - }) - }) - - t.Run("DeleteDataSourceById", func(t *testing.T) { - t.Run("can delete datasource", func(t *testing.T) { - sqlStore := InitTestDB(t) - ds := initDatasource(sqlStore) - - err := sqlStore.DeleteDataSource(context.Background(), &datasources.DeleteDataSourceCommand{ID: ds.Id, OrgID: ds.OrgId}) - require.NoError(t, err) - - query := datasources.GetDataSourcesQuery{OrgId: 10} - err = sqlStore.GetDataSources(context.Background(), &query) - require.NoError(t, err) - - require.Equal(t, 0, len(query.Result)) - }) - - t.Run("Can not delete datasource with wrong orgId", func(t *testing.T) { - sqlStore := InitTestDB(t) - ds := initDatasource(sqlStore) - - err := sqlStore.DeleteDataSource(context.Background(), - &datasources.DeleteDataSourceCommand{ID: ds.Id, OrgID: 123123}) - require.NoError(t, err) - - query := datasources.GetDataSourcesQuery{OrgId: 10} - err = sqlStore.GetDataSources(context.Background(), &query) - require.NoError(t, err) - - require.Equal(t, 1, len(query.Result)) - }) - }) - - t.Run("fires an event when the datasource is deleted", func(t *testing.T) { - sqlStore := InitTestDB(t) - ds := initDatasource(sqlStore) - - var deleted *events.DataSourceDeleted - sqlStore.bus.AddEventListener(func(ctx context.Context, e *events.DataSourceDeleted) error { - deleted = e - return nil - }) - - err := sqlStore.DeleteDataSource(context.Background(), - &datasources.DeleteDataSourceCommand{ID: ds.Id, UID: ds.Uid, Name: ds.Name, OrgID: ds.OrgId}) - require.NoError(t, err) - - require.Eventually(t, func() bool { - return assert.NotNil(t, deleted) - }, time.Second, time.Millisecond) - - require.Equal(t, ds.Id, deleted.ID) - require.Equal(t, ds.OrgId, deleted.OrgID) - require.Equal(t, ds.Name, deleted.Name) - require.Equal(t, ds.Uid, deleted.UID) - }) - - t.Run("does not fire an event when the datasource is not deleted", func(t *testing.T) { - sqlStore := InitTestDB(t) - - var called bool - sqlStore.bus.AddEventListener(func(ctx context.Context, e *events.DataSourceDeleted) error { - called = true - return nil - }) - - err := sqlStore.DeleteDataSource(context.Background(), - &datasources.DeleteDataSourceCommand{ID: 1, UID: "non-existing", Name: "non-existing", OrgID: int64(10)}) - require.NoError(t, err) - - require.Never(t, func() bool { - return called - }, time.Second, time.Millisecond) - }) - - t.Run("DeleteDataSourceByName", func(t *testing.T) { - sqlStore := InitTestDB(t) - ds := initDatasource(sqlStore) - query := datasources.GetDataSourcesQuery{OrgId: 10} - - err := sqlStore.DeleteDataSource(context.Background(), &datasources.DeleteDataSourceCommand{Name: ds.Name, OrgID: ds.OrgId}) - require.NoError(t, err) - - err = sqlStore.GetDataSources(context.Background(), &query) - require.NoError(t, err) - - require.Equal(t, 0, len(query.Result)) - }) - - t.Run("DeleteDataSourceAccessControlPermissions", func(t *testing.T) { - sqlStore := InitTestDB(t) - ds := initDatasource(sqlStore) - - // Init associated permission - errAddPermissions := sqlStore.WithTransactionalDbSession(context.TODO(), func(sess *DBSession) error { - _, err := sess.Table("permission").Insert(ac.Permission{ - RoleID: 1, - Action: "datasources:read", - Scope: ac.Scope("datasources", "id", fmt.Sprintf("%d", ds.Id)), - Updated: time.Now(), - Created: time.Now(), - }) - return err - }) - require.NoError(t, errAddPermissions) - query := datasources.GetDataSourcesQuery{OrgId: 10} - - errDeletingDS := sqlStore.DeleteDataSource(context.Background(), - &datasources.DeleteDataSourceCommand{Name: ds.Name, OrgID: ds.OrgId}, - ) - require.NoError(t, errDeletingDS) - - // Check associated permission - permCount := int64(0) - errGetPermissions := sqlStore.WithTransactionalDbSession(context.TODO(), func(sess *DBSession) error { - var err error - permCount, err = sess.Table("permission").Count() - return err - }) - require.NoError(t, errGetPermissions) - require.Zero(t, permCount, "permissions associated to the data source should have been removed") - - require.Equal(t, 0, len(query.Result)) - }) - - t.Run("GetDataSources", func(t *testing.T) { - t.Run("Number of data sources returned limited to 6 per organization", func(t *testing.T) { - sqlStore := InitTestDB(t) - datasourceLimit := 6 - for i := 0; i < datasourceLimit+1; i++ { - err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{ - OrgId: 10, - Name: "laban" + strconv.Itoa(i), - Type: datasources.DS_GRAPHITE, - Access: datasources.DS_ACCESS_DIRECT, - Url: "http://test", - Database: "site", - ReadOnly: true, - }) - require.NoError(t, err) - } - query := datasources.GetDataSourcesQuery{OrgId: 10, DataSourceLimit: datasourceLimit} - - err := sqlStore.GetDataSources(context.Background(), &query) - - require.NoError(t, err) - require.Equal(t, datasourceLimit, len(query.Result)) - }) - - t.Run("No limit should be applied on the returned data sources if the limit is not set", func(t *testing.T) { - sqlStore := InitTestDB(t) - numberOfDatasource := 5100 - for i := 0; i < numberOfDatasource; i++ { - err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{ - OrgId: 10, - Name: "laban" + strconv.Itoa(i), - Type: datasources.DS_GRAPHITE, - Access: datasources.DS_ACCESS_DIRECT, - Url: "http://test", - Database: "site", - ReadOnly: true, - }) - require.NoError(t, err) - } - query := datasources.GetDataSourcesQuery{OrgId: 10} - - err := sqlStore.GetDataSources(context.Background(), &query) - - require.NoError(t, err) - require.Equal(t, numberOfDatasource, len(query.Result)) - }) - - t.Run("No limit should be applied on the returned data sources if the limit is negative", func(t *testing.T) { - sqlStore := InitTestDB(t) - numberOfDatasource := 5100 - for i := 0; i < numberOfDatasource; i++ { - err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{ - OrgId: 10, - Name: "laban" + strconv.Itoa(i), - Type: datasources.DS_GRAPHITE, - Access: datasources.DS_ACCESS_DIRECT, - Url: "http://test", - Database: "site", - ReadOnly: true, - }) - require.NoError(t, err) - } - query := datasources.GetDataSourcesQuery{OrgId: 10, DataSourceLimit: -1} - - err := sqlStore.GetDataSources(context.Background(), &query) - - require.NoError(t, err) - require.Equal(t, numberOfDatasource, len(query.Result)) - }) - }) - - t.Run("GetDataSourcesByType", func(t *testing.T) { - t.Run("Only returns datasources of specified type", func(t *testing.T) { - sqlStore := InitTestDB(t) - - err := sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{ - OrgId: 10, - Name: "Elasticsearch", - Type: datasources.DS_ES, - Access: datasources.DS_ACCESS_DIRECT, - Url: "http://test", - Database: "site", - ReadOnly: true, - }) - require.NoError(t, err) - - err = sqlStore.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{ - OrgId: 10, - Name: "Graphite", - Type: datasources.DS_GRAPHITE, - Access: datasources.DS_ACCESS_DIRECT, - Url: "http://test", - Database: "site", - ReadOnly: true, - }) - require.NoError(t, err) - - query := datasources.GetDataSourcesByTypeQuery{Type: datasources.DS_ES} - - err = sqlStore.GetDataSourcesByType(context.Background(), &query) - - require.NoError(t, err) - require.Equal(t, 1, len(query.Result)) - }) - - t.Run("Returns an error if no type specified", func(t *testing.T) { - sqlStore := InitTestDB(t) - - query := datasources.GetDataSourcesByTypeQuery{} - - err := sqlStore.GetDataSourcesByType(context.Background(), &query) - - require.Error(t, err) - }) - }) -} - -func TestIntegrationGetDefaultDataSource(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - InitTestDB(t) - - t.Run("should return error if there is no default datasource", func(t *testing.T) { - sqlStore := InitTestDB(t) - - cmd := datasources.AddDataSourceCommand{ - OrgId: 10, - Name: "nisse", - Type: datasources.DS_GRAPHITE, - Access: datasources.DS_ACCESS_DIRECT, - Url: "http://test", - } - - err := sqlStore.AddDataSource(context.Background(), &cmd) - require.NoError(t, err) - - query := datasources.GetDefaultDataSourceQuery{OrgId: 10} - err = sqlStore.GetDefaultDataSource(context.Background(), &query) - require.Error(t, err) - assert.True(t, errors.Is(err, datasources.ErrDataSourceNotFound)) - }) - - t.Run("should return default datasource if exists", func(t *testing.T) { - sqlStore := InitTestDB(t) - - cmd := datasources.AddDataSourceCommand{ - OrgId: 10, - Name: "default datasource", - Type: datasources.DS_GRAPHITE, - Access: datasources.DS_ACCESS_DIRECT, - Url: "http://test", - IsDefault: true, - } - - err := sqlStore.AddDataSource(context.Background(), &cmd) - require.NoError(t, err) - - query := datasources.GetDefaultDataSourceQuery{OrgId: 10} - err = sqlStore.GetDefaultDataSource(context.Background(), &query) - require.NoError(t, err) - assert.Equal(t, "default datasource", query.Result.Name) - }) - - t.Run("should not return default datasource of other organisation", func(t *testing.T) { - sqlStore := InitTestDB(t) - query := datasources.GetDefaultDataSourceQuery{OrgId: 1} - err := sqlStore.GetDefaultDataSource(context.Background(), &query) - require.Error(t, err) - assert.True(t, errors.Is(err, datasources.ErrDataSourceNotFound)) - }) -}