Correlations: Allow creating correlations for provisioned data sources (#73737)

* Allow creating correlations for provisioned data sources

* Update docs

* Fix linting

* Add missing props

* Add missing props

* Fix linting

* Fix linting

* Clarify error name

* Removed error handling for a non-existing use case

* Create a list of deleted data datasources based on all configs

* Add org_id to correlations

* Add tests

* Allow org_id to be null in case org_id=0 is used

* Create organization to ensure stable id is generated

* Fix linting

* Ensure backwards compatibility

* Add deprecation information

* Update comments

* Override existing datasSource variable so the UID is retrieved correctly

* Migrate correlations indices

* Default org_id when migrating

* Remove redundant default

* Make PK non-nullable

* Post merge fixes

* Separate data sources / correlations provisioning

* Adjust comments

* Store new data sources in spy store so it can be used to test correlations as well

* Fix linting

* Update tests

* Ensure response is closed

* Avoid creating duplicates during provisioning

* Fix updating provisioned column and update tests

* Rename error message

* Fix linting errors

* Fix linting errors and rename variable

* Update test

* Update pkg/services/sqlstore/migrations/correlations_mig.go

Co-authored-by: Giordano Ricci <me@giordanoricci.com>

* Remove unused error

* Fix lining

---------

Co-authored-by: Giordano Ricci <me@giordanoricci.com>
This commit is contained in:
Piotr Jamróz
2023-09-13 15:10:09 +02:00
committed by GitHub
parent 38c3483594
commit 946da57b6a
21 changed files with 464 additions and 158 deletions

View File

@@ -196,6 +196,7 @@ Content-Type: application/json
Query parameters:
- **page** - Optional. Specify which page number to return. Use the limit parameter to specify the number of correlations per page. The default is page 1.
- **limit** - Optional. Limits the number of returned correlations per page. The default is 100 correlations per page. The maximum limit is 1000 correlations in a page.
- **sourceUID** - Optional. Specify a source datasource UID to filter by. This can be repeated to filter by multiple datasources.
**Example request:**
@@ -237,6 +238,7 @@ Content-Type: application/json
"sourceUID": "uyBf2637k",
"targetUID": "PDDA8E780A17E7EF1",
"uid": "J6gn7d31L",
"provisioned": false,
"config": {
"type": "query",
"field": "message",
@@ -249,6 +251,7 @@ Content-Type: application/json
"sourceUID": "uyBf2637k",
"targetUID": "P15396BDD62B2BE29",
"uid": "uWCpURgVk",
"provisioned": false,
"config": {
"type": "query",
"field": "message",
@@ -297,6 +300,7 @@ Content-Type: application/json
"sourceUID": "uyBf2637k",
"targetUID": "PDDA8E780A17E7EF1",
"uid": "J6gn7d31L",
"provisioned": false,
"config": {
"type": "query",
"field": "message",
@@ -309,6 +313,7 @@ Content-Type: application/json
"sourceUID": "PDDA8E780A17E7EF1",
"targetUID": "P15396BDD62B2BE29",
"uid": "uWCpURgVk",
"provisioned": false,
"config": {
"type": "query",
"field": "message",

View File

@@ -55,11 +55,6 @@ func (s *CorrelationsService) createHandler(c *contextmodel.ReqContext) response
if errors.Is(err, ErrSourceDataSourceDoesNotExists) || errors.Is(err, ErrTargetDataSourceDoesNotExists) {
return response.Error(http.StatusNotFound, "Data source not found", err)
}
if errors.Is(err, ErrSourceDataSourceReadOnly) {
return response.Error(http.StatusForbidden, "Data source is read only", err)
}
return response.Error(http.StatusInternalServerError, "Failed to add correlation", err)
}
@@ -109,8 +104,8 @@ func (s *CorrelationsService) deleteHandler(c *contextmodel.ReqContext) response
return response.Error(http.StatusNotFound, "Correlation not found", err)
}
if errors.Is(err, ErrSourceDataSourceReadOnly) {
return response.Error(http.StatusForbidden, "Data source is read only", err)
if errors.Is(err, ErrCorrelationReadOnly) {
return response.Error(http.StatusForbidden, "Correlation can only be edited via provisioning", err)
}
return response.Error(http.StatusInternalServerError, "Failed to delete correlation", err)
@@ -170,8 +165,8 @@ func (s *CorrelationsService) updateHandler(c *contextmodel.ReqContext) response
return response.Error(http.StatusNotFound, "Correlation not found", err)
}
if errors.Is(err, ErrSourceDataSourceReadOnly) {
return response.Error(http.StatusForbidden, "Data source is read only", err)
if errors.Is(err, ErrCorrelationReadOnly) {
return response.Error(http.StatusForbidden, "Correlation can only be edited via provisioning", err)
}
return response.Error(http.StatusInternalServerError, "Failed to update correlation", err)

View File

@@ -51,6 +51,7 @@ func ProvideService(sqlStore db.DB, routeRegister routing.RouteRegister, ds data
type Service interface {
CreateCorrelation(ctx context.Context, cmd CreateCorrelationCommand) (Correlation, error)
CreateOrUpdateCorrelation(ctx context.Context, cmd CreateCorrelationCommand) error
DeleteCorrelation(ctx context.Context, cmd DeleteCorrelationCommand) error
DeleteCorrelationsBySourceUID(ctx context.Context, cmd DeleteCorrelationsBySourceUIDCommand) error
DeleteCorrelationsByTargetUID(ctx context.Context, cmd DeleteCorrelationsByTargetUIDCommand) error
@@ -78,6 +79,10 @@ func (s CorrelationsService) CreateCorrelation(ctx context.Context, cmd CreateCo
return s.createCorrelation(ctx, cmd)
}
func (s CorrelationsService) CreateOrUpdateCorrelation(ctx context.Context, cmd CreateCorrelationCommand) error {
return s.createOrUpdateCorrelation(ctx, cmd)
}
func (s CorrelationsService) DeleteCorrelation(ctx context.Context, cmd DeleteCorrelationCommand) error {
return s.deleteCorrelation(ctx, cmd)
}

View File

@@ -3,6 +3,8 @@ package correlations
import (
"context"
"xorm.io/core"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/quota"
@@ -19,6 +21,7 @@ func (s CorrelationsService) createCorrelation(ctx context.Context, cmd CreateCo
Label: cmd.Label,
Description: cmd.Description,
Config: cmd.Config,
Provisioned: cmd.Provisioned,
}
err := s.SQLStore.WithTransactionalDbSession(ctx, func(session *db.Session) error {
@@ -28,15 +31,11 @@ func (s CorrelationsService) createCorrelation(ctx context.Context, cmd CreateCo
OrgID: cmd.OrgId,
UID: cmd.SourceUID,
}
dataSource, err := s.DataSourceService.GetDataSource(ctx, query)
_, err = s.DataSourceService.GetDataSource(ctx, query)
if err != nil {
return ErrSourceDataSourceDoesNotExists
}
if !cmd.SkipReadOnlyCheck && dataSource.ReadOnly {
return ErrSourceDataSourceReadOnly
}
if cmd.TargetUID != nil {
if _, err = s.DataSourceService.GetDataSource(ctx, &datasources.GetDataSourceQuery{
OrgID: cmd.OrgId,
@@ -67,13 +66,19 @@ func (s CorrelationsService) deleteCorrelation(ctx context.Context, cmd DeleteCo
OrgID: cmd.OrgId,
UID: cmd.SourceUID,
}
dataSource, err := s.DataSourceService.GetDataSource(ctx, query)
_, err := s.DataSourceService.GetDataSource(ctx, query)
if err != nil {
return ErrSourceDataSourceDoesNotExists
}
if dataSource.ReadOnly {
return ErrSourceDataSourceReadOnly
correlation, err := s.GetCorrelation(ctx, GetCorrelationQuery(cmd))
if err != nil {
return err
}
if correlation.Provisioned {
return ErrCorrelationReadOnly
}
deletedCount, err := session.Delete(&Correlation{UID: cmd.UID, SourceUID: cmd.SourceUID})
@@ -96,15 +101,11 @@ func (s CorrelationsService) updateCorrelation(ctx context.Context, cmd UpdateCo
OrgID: cmd.OrgId,
UID: cmd.SourceUID,
}
dataSource, err := s.DataSourceService.GetDataSource(ctx, query)
_, err := s.DataSourceService.GetDataSource(ctx, query)
if err != nil {
return ErrSourceDataSourceDoesNotExists
}
if dataSource.ReadOnly {
return ErrSourceDataSourceReadOnly
}
found, err := session.Get(&correlation)
if !found {
return ErrCorrelationNotFound
@@ -112,6 +113,9 @@ func (s CorrelationsService) updateCorrelation(ctx context.Context, cmd UpdateCo
if err != nil {
return err
}
if correlation.Provisioned {
return ErrCorrelationReadOnly
}
if cmd.Label != nil {
correlation.Label = *cmd.Label
@@ -270,7 +274,13 @@ func (s CorrelationsService) getCorrelations(ctx context.Context, cmd GetCorrela
func (s CorrelationsService) deleteCorrelationsBySourceUID(ctx context.Context, cmd DeleteCorrelationsBySourceUIDCommand) error {
return s.SQLStore.WithDbSession(ctx, func(session *db.Session) error {
// Correlations created before the fix #72498 may have org_id = 0, but it's deprecated and will be removed in #72325
_, err := session.Where("source_uid = ? and (org_id = ? or org_id = 0)", cmd.SourceUID, cmd.OrgId).Delete(&Correlation{})
db := session.Where("source_uid = ? and (org_id = ? or org_id = 0)", cmd.SourceUID, cmd.OrgId)
if cmd.OnlyProvisioned {
// bool in a struct needs to be in Where
// https://github.com/go-xorm/xorm/blob/v0.7.9/engine_cond.go#L102
db = db.And("provisioned = ?", true)
}
_, err := db.Delete(&Correlation{})
return err
})
}
@@ -282,3 +292,38 @@ func (s CorrelationsService) deleteCorrelationsByTargetUID(ctx context.Context,
return err
})
}
// internal use: It's require only for correct migration of existing records. Can be removed in Grafana 11.
func (s CorrelationsService) createOrUpdateCorrelation(ctx context.Context, cmd CreateCorrelationCommand) error {
correlation := Correlation{
SourceUID: cmd.SourceUID,
OrgID: cmd.OrgId,
TargetUID: cmd.TargetUID,
Label: cmd.Label,
Description: cmd.Description,
Config: cmd.Config,
Provisioned: false,
}
found := false
err := s.SQLStore.WithDbSession(ctx, func(session *db.Session) error {
has, err := session.Get(&correlation)
found = has
return err
})
if err != nil {
return err
}
if found && cmd.Provisioned {
correlation.Provisioned = true
return s.SQLStore.WithDbSession(ctx, func(session *db.Session) error {
_, err := session.ID(core.NewPK(correlation.UID, correlation.SourceUID, correlation.OrgID)).Cols("provisioned").Update(&correlation)
return err
})
} else {
_, err := s.createCorrelation(ctx, cmd)
return err
}
}

View File

@@ -9,18 +9,17 @@ import (
)
var (
ErrSourceDataSourceReadOnly = errors.New("source data source is read only")
ErrSourceDataSourceDoesNotExists = errors.New("source data source does not exist")
ErrTargetDataSourceDoesNotExists = errors.New("target data source does not exist")
ErrCorrelationFailedGenerateUniqueUid = errors.New("failed to generate unique correlation UID")
ErrCorrelationNotFound = errors.New("correlation not found")
ErrUpdateCorrelationEmptyParams = errors.New("not enough parameters to edit correlation")
ErrInvalidConfigType = errors.New("invalid correlation config type")
ErrInvalidTransformationType = errors.New("invalid transformation type")
ErrTransformationNotNested = errors.New("transformations must be nested under config")
ErrTransformationRegexReqExp = errors.New("regex transformations require expression")
ErrCorrelationsQuotaFailed = errors.New("error getting correlations quota")
ErrCorrelationsQuotaReached = errors.New("correlations quota reached")
ErrCorrelationReadOnly = errors.New("correlation can only be edited via provisioning")
ErrSourceDataSourceDoesNotExists = errors.New("source data source does not exist")
ErrTargetDataSourceDoesNotExists = errors.New("target data source does not exist")
ErrCorrelationNotFound = errors.New("correlation not found")
ErrUpdateCorrelationEmptyParams = errors.New("not enough parameters to edit correlation")
ErrInvalidConfigType = errors.New("invalid correlation config type")
ErrInvalidTransformationType = errors.New("invalid transformation type")
ErrTransformationNotNested = errors.New("transformations must be nested under config")
ErrTransformationRegexReqExp = errors.New("regex transformations require expression")
ErrCorrelationsQuotaFailed = errors.New("error getting correlations quota")
ErrCorrelationsQuotaReached = errors.New("correlations quota reached")
)
const (
@@ -123,6 +122,8 @@ type Correlation struct {
Description string `json:"description" xorm:"description"`
// Correlation Configuration
Config CorrelationConfig `json:"config" xorm:"jsonb config"`
// Provisioned True if the correlation was created during provisioning
Provisioned bool `json:"provisioned"`
}
type GetCorrelationsResponseBody struct {
@@ -144,9 +145,8 @@ type CreateCorrelationResponseBody struct {
// swagger:model
type CreateCorrelationCommand struct {
// UID of the data source for which correlation is created.
SourceUID string `json:"-"`
OrgId int64 `json:"-"`
SkipReadOnlyCheck bool `json:"-"`
SourceUID string `json:"-"`
OrgId int64 `json:"-"`
// Target data source UID to which the correlation is created. required if config.type = query
// example: PE1C5CBDA0504A6A3
TargetUID *string `json:"targetUID"`
@@ -158,6 +158,8 @@ type CreateCorrelationCommand struct {
Description string `json:"description"`
// Arbitrary configuration object handled in frontend
Config CorrelationConfig `json:"config" binding:"Required"`
// True if correlation was created with provisioning. This makes it read-only.
Provisioned bool `json:"provisioned"`
}
func (c CreateCorrelationCommand) Validate() error {
@@ -288,8 +290,9 @@ type GetCorrelationsQuery struct {
}
type DeleteCorrelationsBySourceUIDCommand struct {
SourceUID string
OrgId int64
SourceUID string
OrgId int64
OnlyProvisioned bool
}
type DeleteCorrelationsByTargetUIDCommand struct {

View File

@@ -149,6 +149,12 @@ type DeleteDataSourceCommand struct {
DeletedDatasourcesCount int64
UpdateSecretFn UpdateSecretFn
// Optional way to skip publishing delete event for data sources that are
// deleted just to be re-created with the same UID during provisioning.
// In such case we don't want to publish the event that triggers clean-up
// of related resources (like correlations)
SkipPublish bool
}
// Function for updating secrets along with datasources, to ensure atomicity

View File

@@ -165,7 +165,7 @@ func (ss *SqlStore) DeleteDataSource(ctx context.Context, cmd *datasources.Delet
}
// Publish data source deletion event
if cmd.DeletedDatasourcesCount > 0 {
if cmd.DeletedDatasourcesCount > 0 && !cmd.SkipPublish {
sess.PublishAfterCommit(&events.DataSourceDeleted{
Timestamp: time.Now(),
Name: ds.Name,

View File

@@ -20,6 +20,7 @@ var (
twoDatasourcesConfig = "testdata/two-datasources"
twoDatasourcesConfigPurgeOthers = "testdata/insert-two-delete-two"
deleteOneDatasource = "testdata/delete-one"
recreateOneDatasource = "testdata/recreate-one"
doubleDatasourcesConfig = "testdata/double-default"
allProperties = "testdata/all-properties"
versionZero = "testdata/version-0"
@@ -249,8 +250,10 @@ func TestDatasourceAsConfig(t *testing.T) {
}
require.Equal(t, 2, len(correlationsStore.created))
require.Equal(t, 0, len(correlationsStore.deletedBySourceUID))
require.Equal(t, 0, len(correlationsStore.deletedByTargetUID))
// clean-up of provisioned correlations called once per inserted/updated data source
require.Equal(t, 1, len(store.inserted))
require.Equal(t, 1, len(correlationsStore.deletedBySourceUID))
require.Equal(t, true, correlationsStore.deletedBySourceUID[0].OnlyProvisioned)
})
t.Run("Updating existing datasource deletes existing correlations and creates two", func(t *testing.T) {
@@ -264,8 +267,10 @@ func TestDatasourceAsConfig(t *testing.T) {
}
require.Equal(t, 2, len(correlationsStore.created))
// clean-up of provisioned correlations called once per inserted/updated data source
require.Equal(t, 1, len(store.updated))
require.Equal(t, 1, len(correlationsStore.deletedBySourceUID))
require.Equal(t, 0, len(correlationsStore.deletedByTargetUID))
require.Equal(t, true, correlationsStore.deletedBySourceUID[0].OnlyProvisioned)
})
t.Run("Deleting datasource deletes existing correlations", func(t *testing.T) {
@@ -280,8 +285,34 @@ func TestDatasourceAsConfig(t *testing.T) {
}
require.Equal(t, 0, len(correlationsStore.created))
require.Equal(t, 1, len(store.deleted))
// publish event is not skipped because data source is actually deleted
require.Equal(t, false, store.deleted[0].SkipPublish)
})
t.Run("Re-creating datasource does not delete existing correlations", func(t *testing.T) {
store := &spyStore{items: []*datasources.DataSource{{Name: "Test", OrgID: 1, UID: "test"}}}
orgFake := &orgtest.FakeOrgService{}
targetUid := "target-uid"
correlationsStore := &mockCorrelationsStore{items: []correlations.Correlation{{UID: "some-uid", SourceUID: "some-uid", TargetUID: &targetUid}}}
dc := newDatasourceProvisioner(logger, store, correlationsStore, orgFake)
err := dc.applyChanges(context.Background(), recreateOneDatasource)
if err != nil {
t.Fatalf("applyChanges return an error %v", err)
}
require.Equal(t, 0, len(correlationsStore.created))
require.Equal(t, 1, len(store.deleted))
// publish event is skipped because...
require.Equal(t, true, store.deleted[0].SkipPublish)
// ... the data source is re-created...
require.Equal(t, 1, len(store.inserted))
require.Equal(t, store.deleted[0].Name, store.inserted[0].Name)
// correlations for provisioned data sources are re-recreated
require.Equal(t, 1, len(correlationsStore.deletedBySourceUID))
require.Equal(t, 1, len(correlationsStore.deletedByTargetUID))
require.Equal(t, true, correlationsStore.deletedBySourceUID[0].OnlyProvisioned)
})
t.Run("Using correct organization id", func(t *testing.T) {
@@ -295,13 +326,11 @@ func TestDatasourceAsConfig(t *testing.T) {
}
require.Equal(t, 2, len(correlationsStore.created))
// triggered twice - clean up on delete + update (because of the store setup above)
require.Equal(t, 2, len(correlationsStore.deletedBySourceUID))
require.Equal(t, int64(2), correlationsStore.deletedBySourceUID[0].OrgId)
// triggered for each provisioned data source
require.Equal(t, 3, len(correlationsStore.deletedBySourceUID))
require.Equal(t, int64(1), correlationsStore.deletedBySourceUID[0].OrgId)
require.Equal(t, int64(2), correlationsStore.deletedBySourceUID[1].OrgId)
// triggered once - just the clean up
require.Equal(t, 1, len(correlationsStore.deletedByTargetUID))
require.Equal(t, int64(2), correlationsStore.deletedByTargetUID[0].OrgId)
require.Equal(t, int64(3), correlationsStore.deletedBySourceUID[2].OrgId)
})
})
}
@@ -369,6 +398,11 @@ func (m *mockCorrelationsStore) CreateCorrelation(c context.Context, cmd correla
return correlations.Correlation{}, nil
}
func (m *mockCorrelationsStore) CreateOrUpdateCorrelation(c context.Context, cmd correlations.CreateCorrelationCommand) error {
m.created = append(m.created, cmd)
return nil
}
func (m *mockCorrelationsStore) DeleteCorrelationsBySourceUID(c context.Context, cmd correlations.DeleteCorrelationsBySourceUIDCommand) error {
m.deletedBySourceUID = append(m.deletedBySourceUID, cmd)
return nil
@@ -397,9 +431,10 @@ func (s *spyStore) GetDataSource(ctx context.Context, query *datasources.GetData
func (s *spyStore) DeleteDataSource(ctx context.Context, cmd *datasources.DeleteDataSourceCommand) error {
s.deleted = append(s.deleted, cmd)
for _, v := range s.items {
for i, v := range s.items {
if cmd.Name == v.Name && cmd.OrgID == v.OrgID {
cmd.DeletedDatasourcesCount = 1
s.items = append(s.items[:i], s.items[i+1:]...)
return nil
}
}
@@ -408,7 +443,9 @@ func (s *spyStore) DeleteDataSource(ctx context.Context, cmd *datasources.Delete
func (s *spyStore) AddDataSource(ctx context.Context, cmd *datasources.AddDataSourceCommand) (*datasources.DataSource, error) {
s.inserted = append(s.inserted, cmd)
return &datasources.DataSource{UID: cmd.UID}, nil
newDataSource := &datasources.DataSource{UID: cmd.UID, Name: cmd.Name, OrgID: cmd.OrgID}
s.items = append(s.items, newDataSource)
return newDataSource, nil
}
func (s *spyStore) UpdateDataSource(ctx context.Context, cmd *datasources.UpdateDataSourceCommand) (*datasources.DataSource, error) {

View File

@@ -5,12 +5,11 @@ import (
"errors"
"fmt"
jsoniter "github.com/json-iterator/go"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/correlations"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/org"
jsoniter "github.com/json-iterator/go"
)
type Store interface {
@@ -24,6 +23,7 @@ type CorrelationsStore interface {
DeleteCorrelationsByTargetUID(ctx context.Context, cmd correlations.DeleteCorrelationsByTargetUIDCommand) error
DeleteCorrelationsBySourceUID(ctx context.Context, cmd correlations.DeleteCorrelationsBySourceUIDCommand) error
CreateCorrelation(ctx context.Context, cmd correlations.CreateCorrelationCommand) (correlations.Correlation, error)
CreateOrUpdateCorrelation(ctx context.Context, cmd correlations.CreateCorrelationCommand) error
}
var (
@@ -57,13 +57,11 @@ func newDatasourceProvisioner(log log.Logger, store Store, correlationsStore Cor
}
}
func (dc *DatasourceProvisioner) apply(ctx context.Context, cfg *configs) error {
if err := dc.deleteDatasources(ctx, cfg.DeleteDatasources); err != nil {
func (dc *DatasourceProvisioner) provisionDataSources(ctx context.Context, cfg *configs, willExistAfterProvisioning map[DataSourceMapKey]bool) error {
if err := dc.deleteDatasources(ctx, cfg.DeleteDatasources, willExistAfterProvisioning); err != nil {
return err
}
correlationsToInsert := make([]correlations.CreateCorrelationCommand, 0)
for _, ds := range cfg.Datasources {
cmd := &datasources.GetDataSourceQuery{OrgID: ds.OrgID, Name: ds.Name}
dataSource, err := dc.store.GetDataSource(ctx, cmd)
@@ -74,63 +72,91 @@ func (dc *DatasourceProvisioner) apply(ctx context.Context, cfg *configs) error
if errors.Is(err, datasources.ErrDataSourceNotFound) {
insertCmd := createInsertCommand(ds)
dc.log.Info("inserting datasource from configuration ", "name", insertCmd.Name, "uid", insertCmd.UID)
dataSource, err := dc.store.AddDataSource(ctx, insertCmd)
_, err = dc.store.AddDataSource(ctx, insertCmd)
if err != nil {
return err
}
for _, correlation := range ds.Correlations {
if insertCorrelationCmd, err := makeCreateCorrelationCommand(correlation, dataSource.UID, insertCmd.OrgID); err == nil {
correlationsToInsert = append(correlationsToInsert, insertCorrelationCmd)
} else {
dc.log.Error("failed to parse correlation", "correlation", correlation)
return err
}
}
} else {
updateCmd := createUpdateCommand(ds, dataSource.ID)
dc.log.Debug("updating datasource from configuration", "name", updateCmd.Name, "uid", updateCmd.UID)
if _, err := dc.store.UpdateDataSource(ctx, updateCmd); err != nil {
return err
}
if len(ds.Correlations) > 0 {
if err := dc.correlationsStore.DeleteCorrelationsBySourceUID(ctx, correlations.DeleteCorrelationsBySourceUIDCommand{
SourceUID: dataSource.UID,
OrgId: dataSource.OrgID,
}); err != nil {
return err
}
}
for _, correlation := range ds.Correlations {
if insertCorrelationCmd, err := makeCreateCorrelationCommand(correlation, dataSource.UID, updateCmd.OrgID); err == nil {
correlationsToInsert = append(correlationsToInsert, insertCorrelationCmd)
} else {
dc.log.Error("failed to parse correlation", "correlation", correlation)
return err
}
}
}
}
for _, createCorrelationCmd := range correlationsToInsert {
if _, err := dc.correlationsStore.CreateCorrelation(ctx, createCorrelationCmd); err != nil {
return fmt.Errorf("err=%s source=%s", err.Error(), createCorrelationCmd.SourceUID)
}
}
return nil
}
func (dc *DatasourceProvisioner) provisionCorrelations(ctx context.Context, cfg *configs) error {
for _, ds := range cfg.Datasources {
cmd := &datasources.GetDataSourceQuery{OrgID: ds.OrgID, Name: ds.Name}
dataSource, err := dc.store.GetDataSource(ctx, cmd)
if errors.Is(err, datasources.ErrDataSourceNotFound) {
return err
}
if err := dc.correlationsStore.DeleteCorrelationsBySourceUID(ctx, correlations.DeleteCorrelationsBySourceUIDCommand{
SourceUID: dataSource.UID,
OrgId: dataSource.OrgID,
OnlyProvisioned: true,
}); err != nil {
return err
}
for _, correlation := range ds.Correlations {
createCorrelationCmd, err := makeCreateCorrelationCommand(correlation, dataSource.UID, dataSource.OrgID)
if err != nil {
dc.log.Error("failed to parse correlation", "correlation", correlation)
return err
}
// "Provisioned" column was introduced in #71110. Any records that were created before this change
// are marked as "not provisioned". To avoid duplicates we ensure these records are updated instead
// of being inserted once again with Provisioned=true.
// This is required to help users upgrade with confidence. Post GA we do not expect this code to be
// needed at all as it should result in a no-op. This should be mentioned in what's new docs when
// feature becomes GA.
// This can be changed to dc.correlationsStore.CreateCorrelation in Grafana 11 and CreateOrUpdateCorrelation
// can be removed.
if err := dc.correlationsStore.CreateOrUpdateCorrelation(ctx, createCorrelationCmd); err != nil {
return fmt.Errorf("err=%s source=%s", err.Error(), createCorrelationCmd.SourceUID)
}
}
}
return nil
}
type DataSourceMapKey struct {
Name string
OrgId int64
}
func (dc *DatasourceProvisioner) applyChanges(ctx context.Context, configPath string) error {
configs, err := dc.cfgProvider.readConfig(ctx, configPath)
if err != nil {
return err
}
// Creates a list of data sources that will be ultimately deleted after provisioning finishes
willExistAfterProvisioning := map[DataSourceMapKey]bool{}
for _, cfg := range configs {
if err := dc.apply(ctx, cfg); err != nil {
for _, ds := range cfg.DeleteDatasources {
willExistAfterProvisioning[DataSourceMapKey{Name: ds.Name, OrgId: ds.OrgID}] = false
}
for _, ds := range cfg.Datasources {
willExistAfterProvisioning[DataSourceMapKey{Name: ds.Name, OrgId: ds.OrgID}] = true
}
}
for _, cfg := range configs {
if err := dc.provisionDataSources(ctx, cfg, willExistAfterProvisioning); err != nil {
return err
}
}
for _, cfg := range configs {
if err := dc.provisionCorrelations(ctx, cfg); err != nil {
return err
}
}
@@ -141,11 +167,11 @@ func (dc *DatasourceProvisioner) applyChanges(ctx context.Context, configPath st
func makeCreateCorrelationCommand(correlation map[string]any, SourceUID string, OrgId int64) (correlations.CreateCorrelationCommand, error) {
var json = jsoniter.ConfigCompatibleWithStandardLibrary
createCommand := correlations.CreateCorrelationCommand{
SourceUID: SourceUID,
Label: correlation["label"].(string),
Description: correlation["description"].(string),
OrgId: OrgId,
SkipReadOnlyCheck: true,
SourceUID: SourceUID,
Label: correlation["label"].(string),
Description: correlation["description"].(string),
OrgId: OrgId,
Provisioned: true,
}
targetUID, ok := correlation["targetUID"].(string)
@@ -182,37 +208,23 @@ func makeCreateCorrelationCommand(correlation map[string]any, SourceUID string,
return createCommand, nil
}
func (dc *DatasourceProvisioner) deleteDatasources(ctx context.Context, dsToDelete []*deleteDatasourceConfig) error {
func (dc *DatasourceProvisioner) deleteDatasources(ctx context.Context, dsToDelete []*deleteDatasourceConfig, willExistAfterProvisioning map[DataSourceMapKey]bool) error {
for _, ds := range dsToDelete {
cmd := &datasources.DeleteDataSourceCommand{OrgID: ds.OrgID, Name: ds.Name}
getDsQuery := &datasources.GetDataSourceQuery{Name: ds.Name, OrgID: ds.OrgID}
dataSource, err := dc.store.GetDataSource(ctx, getDsQuery)
_, err := dc.store.GetDataSource(ctx, getDsQuery)
if err != nil && !errors.Is(err, datasources.ErrDataSourceNotFound) {
return err
}
// Skip publishing the event as the data source is not really deleted, it will be re-created during provisioning
// This is to avoid cleaning up any resources related to the data source (e.g. correlations)
skipPublish := willExistAfterProvisioning[DataSourceMapKey{Name: ds.Name, OrgId: ds.OrgID}]
cmd := &datasources.DeleteDataSourceCommand{OrgID: ds.OrgID, Name: ds.Name, SkipPublish: skipPublish}
if err := dc.store.DeleteDataSource(ctx, cmd); err != nil {
return err
}
if dataSource != nil {
if err := dc.correlationsStore.DeleteCorrelationsBySourceUID(ctx, correlations.DeleteCorrelationsBySourceUIDCommand{
SourceUID: dataSource.UID,
OrgId: dataSource.OrgID,
}); err != nil {
return err
}
if err := dc.correlationsStore.DeleteCorrelationsByTargetUID(ctx, correlations.DeleteCorrelationsByTargetUIDCommand{
TargetUID: dataSource.UID,
OrgId: dataSource.OrgID,
}); err != nil {
return err
}
dc.log.Info("deleted correlations based on configuration", "ds_name", ds.Name)
}
if cmd.DeletedDatasourcesCount > 0 {
dc.log.Info("deleted datasource based on configuration", "name", ds.Name)
}

View File

@@ -0,0 +1,9 @@
apiVersion: 1
deleteDatasources:
- name: Test
datasources:
- name: Test
uid: test
type: type

View File

@@ -59,4 +59,8 @@ func addCorrelationsMigrations(mg *Migrator) {
"description": "description",
"config": "config",
})
mg.AddMigration("add provisioning column", NewAddColumnMigration(correlationsV2, &Column{
Name: "provisioned", Type: DB_Bool, Nullable: false, Default: "0",
}))
}

View File

@@ -182,3 +182,10 @@ func (c TestContext) createCorrelation(cmd correlations.CreateCorrelationCommand
require.NoError(c.t, err)
return correlation
}
func (c TestContext) createOrUpdateCorrelation(cmd correlations.CreateCorrelationCommand) {
c.t.Helper()
err := c.env.Server.HTTPServer.CorrelationsService.CreateOrUpdateCorrelation(context.Background(), cmd)
require.NoError(c.t, err)
}

View File

@@ -166,7 +166,7 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
require.NoError(t, res.Body.Close())
})
t.Run("creating a correlation originating from a read-only data source should result in a 403", func(t *testing.T) {
t.Run("creating a correlation originating from a read-only data source should work", func(t *testing.T) {
res := ctx.Post(PostParams{
url: fmt.Sprintf("/api/datasources/uid/%s/correlations", readOnlyDS),
body: fmt.Sprintf(`{
@@ -179,17 +179,20 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
}`, readOnlyDS),
user: adminUser,
})
require.Equal(t, http.StatusForbidden, res.StatusCode)
require.Equal(t, http.StatusOK, res.StatusCode)
responseBody, err := io.ReadAll(res.Body)
require.NoError(t, err)
var response errorResponseBody
var response correlations.CreateCorrelationResponseBody
err = json.Unmarshal(responseBody, &response)
require.NoError(t, err)
require.Equal(t, "Data source is read only", response.Message)
require.Equal(t, correlations.ErrSourceDataSourceReadOnly.Error(), response.Error)
require.Equal(t, "Correlation created", response.Message)
require.Equal(t, readOnlyDS, response.Result.SourceUID)
require.Equal(t, readOnlyDS, *response.Result.TargetUID)
require.Equal(t, "", response.Result.Description)
require.Equal(t, "", response.Result.Label)
require.NoError(t, res.Body.Close())
})

View File

@@ -50,6 +50,7 @@ func TestIntegrationDeleteCorrelation(t *testing.T) {
}
dataSource = ctx.createDs(createDsCommand)
writableDs := dataSource.UID
writableDsId := dataSource.ID
writableDsOrgId := dataSource.OrgID
t.Run("Unauthenticated users shouldn't be able to delete correlations", func(t *testing.T) {
@@ -130,9 +131,16 @@ func TestIntegrationDeleteCorrelation(t *testing.T) {
require.NoError(t, res.Body.Close())
})
t.Run("deleting a correlation originating from a read-only data source should result in a 403", func(t *testing.T) {
t.Run("deleting a read-only correlation should result in a 403", func(t *testing.T) {
correlation := ctx.createCorrelation(correlations.CreateCorrelationCommand{
SourceUID: writableDs,
TargetUID: &writableDs,
OrgId: writableDsOrgId,
Provisioned: true,
})
res := ctx.Delete(DeleteParams{
url: fmt.Sprintf("/api/datasources/uid/%s/correlations/%s", readOnlyDS, "nonexistent-correlation-uid"),
url: fmt.Sprintf("/api/datasources/uid/%s/correlations/%s", correlation.SourceUID, correlation.UID),
user: adminUser,
})
require.Equal(t, http.StatusForbidden, res.StatusCode)
@@ -144,8 +152,8 @@ func TestIntegrationDeleteCorrelation(t *testing.T) {
err = json.Unmarshal(responseBody, &response)
require.NoError(t, err)
require.Equal(t, "Data source is read only", response.Message)
require.Equal(t, correlations.ErrSourceDataSourceReadOnly.Error(), response.Error)
require.Equal(t, "Correlation can only be edited via provisioning", response.Message)
require.Equal(t, correlations.ErrCorrelationReadOnly.Error(), response.Error)
require.NoError(t, res.Body.Close())
})
@@ -213,4 +221,45 @@ func TestIntegrationDeleteCorrelation(t *testing.T) {
require.NoError(t, res.Body.Close())
require.Equal(t, http.StatusNotFound, res.StatusCode)
})
t.Run("deleting data source removes related correlations", func(t *testing.T) {
ctx.createCorrelation(correlations.CreateCorrelationCommand{
SourceUID: writableDs,
TargetUID: &readOnlyDS,
OrgId: writableDsOrgId,
Provisioned: false,
})
ctx.createCorrelation(correlations.CreateCorrelationCommand{
SourceUID: writableDs,
TargetUID: &readOnlyDS,
OrgId: writableDsOrgId,
Provisioned: true,
})
res := ctx.Delete(DeleteParams{
url: fmt.Sprintf("/api/datasources/%d", writableDsId),
user: adminUser,
})
require.Equal(t, http.StatusOK, res.StatusCode)
require.NoError(t, res.Body.Close())
res = ctx.Get(GetParams{
url: "/api/datasources/correlations",
user: adminUser,
page: "0",
})
require.Equal(t, http.StatusOK, res.StatusCode)
responseBody, err := io.ReadAll(res.Body)
require.NoError(t, err)
var response correlations.GetCorrelationsResponseBody
err = json.Unmarshal(responseBody, &response)
require.NoError(t, err)
require.Len(t, response.Correlations, 0)
require.NoError(t, res.Body.Close())
})
}

View File

@@ -0,0 +1,120 @@
package correlations
import (
"encoding/json"
"io"
"net/http"
"testing"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/services/correlations"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/services/user"
)
func TestIntegrationCreateOrUpdateCorrelation(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
ctx := NewTestEnv(t)
adminUser := ctx.createUser(user.CreateUserCommand{
DefaultOrgRole: string(org.RoleAdmin),
Password: "admin",
Login: "admin",
})
createDsCommand := &datasources.AddDataSourceCommand{
Name: "loki",
Type: "loki",
OrgID: adminUser.User.OrgID,
}
dataSource := ctx.createDs(createDsCommand)
needsMigration := ctx.createCorrelation(correlations.CreateCorrelationCommand{
SourceUID: dataSource.UID,
TargetUID: &dataSource.UID,
OrgId: dataSource.OrgID,
Label: "needs migration",
Config: correlations.CorrelationConfig{
Type: correlations.ConfigTypeQuery,
Field: "foo",
Target: map[string]any{},
Transformations: []correlations.Transformation{
{Type: "logfmt"},
},
},
Provisioned: false,
})
ctx.createCorrelation(correlations.CreateCorrelationCommand{
SourceUID: dataSource.UID,
TargetUID: &dataSource.UID,
OrgId: dataSource.OrgID,
Label: "existing",
Config: correlations.CorrelationConfig{
Type: correlations.ConfigTypeQuery,
Field: "foo",
Target: map[string]any{},
Transformations: []correlations.Transformation{
{Type: "logfmt"},
},
},
Provisioned: false,
})
t.Run("Correctly marks existing correlations as provisioned", func(t *testing.T) {
// should be updated
ctx.createOrUpdateCorrelation(correlations.CreateCorrelationCommand{
SourceUID: needsMigration.SourceUID,
OrgId: needsMigration.OrgID,
TargetUID: needsMigration.TargetUID,
Label: needsMigration.Label,
Description: needsMigration.Description,
Config: needsMigration.Config,
Provisioned: true,
})
// should be added
ctx.createOrUpdateCorrelation(correlations.CreateCorrelationCommand{
SourceUID: needsMigration.SourceUID,
OrgId: needsMigration.OrgID,
TargetUID: needsMigration.TargetUID,
Label: "different",
Description: needsMigration.Description,
Config: needsMigration.Config,
Provisioned: true,
})
res := ctx.Get(GetParams{
url: "/api/datasources/correlations",
user: adminUser,
})
require.Equal(t, http.StatusOK, res.StatusCode)
responseBody, err := io.ReadAll(res.Body)
require.NoError(t, err)
var response correlations.GetCorrelationsResponseBody
err = json.Unmarshal(responseBody, &response)
require.NoError(t, err)
require.Len(t, response.Correlations, 3)
unordered := make(map[string]correlations.Correlation)
for _, v := range response.Correlations {
unordered[v.Label] = v
}
// existing correlation is updated
require.EqualValues(t, true, unordered["needs migration"].Provisioned)
// other existing correlations are not changed
require.EqualValues(t, false, unordered["existing"].Provisioned)
// new correlation is added
require.EqualValues(t, true, unordered["different"].Provisioned)
require.NoError(t, res.Body.Close())
})
}

View File

@@ -35,20 +35,11 @@ func TestIntegrationUpdateCorrelation(t *testing.T) {
})
createDsCommand := &datasources.AddDataSourceCommand{
Name: "read-only",
Type: "loki",
ReadOnly: true,
OrgID: adminUser.User.OrgID,
}
dataSource := ctx.createDs(createDsCommand)
readOnlyDS := dataSource.UID
createDsCommand = &datasources.AddDataSourceCommand{
Name: "writable",
Type: "loki",
OrgID: adminUser.User.OrgID,
}
dataSource = ctx.createDs(createDsCommand)
dataSource := ctx.createDs(createDsCommand)
writableDs := dataSource.UID
writableDsOrgId := dataSource.OrgID
@@ -137,9 +128,16 @@ func TestIntegrationUpdateCorrelation(t *testing.T) {
require.NoError(t, res.Body.Close())
})
t.Run("updating a correlation originating from a read-only data source should result in a 403", func(t *testing.T) {
t.Run("updating a read-only correlation should result in a 403", func(t *testing.T) {
correlation := ctx.createCorrelation(correlations.CreateCorrelationCommand{
SourceUID: writableDs,
TargetUID: &writableDs,
OrgId: writableDsOrgId,
Provisioned: true,
})
res := ctx.Patch(PatchParams{
url: fmt.Sprintf("/api/datasources/uid/%s/correlations/%s", readOnlyDS, "nonexistent-correlation-uid"),
url: fmt.Sprintf("/api/datasources/uid/%s/correlations/%s", correlation.SourceUID, correlation.UID),
user: adminUser,
body: `{
"label": "some-label"
@@ -154,8 +152,8 @@ func TestIntegrationUpdateCorrelation(t *testing.T) {
err = json.Unmarshal(responseBody, &response)
require.NoError(t, err)
require.Equal(t, "Data source is read only", response.Message)
require.Equal(t, correlations.ErrSourceDataSourceReadOnly.Error(), response.Error)
require.Equal(t, "Correlation can only be edited via provisioning", response.Message)
require.Equal(t, correlations.ErrCorrelationReadOnly.Error(), response.Error)
require.NoError(t, res.Body.Close())
})

View File

@@ -47,7 +47,7 @@ const renderWithContext = async (
const matches = url.match(/^\/api\/datasources\/uid\/(?<sourceUID>[a-zA-Z0-9]+)\/correlations$/);
if (matches?.groups) {
const { sourceUID } = matches.groups;
const correlation = { sourceUID, ...data, uid: uniqueId() };
const correlation = { sourceUID, ...data, uid: uniqueId(), provisioned: false };
correlations.push(correlation);
return createCreateCorrelationResponse(correlation);
}
@@ -66,7 +66,7 @@ const renderWithContext = async (
}
return c;
});
return createUpdateCorrelationResponse({ sourceUID, ...data, uid: uniqueId() });
return createUpdateCorrelationResponse({ sourceUID, ...data, uid: uniqueId(), provisioned: false });
}
throw createFetchCorrelationsError();
@@ -358,6 +358,7 @@ describe('CorrelationsPage', () => {
targetUID: 'loki',
uid: '1',
label: 'Some label',
provisioned: false,
config: {
field: 'line',
target: {},
@@ -373,6 +374,7 @@ describe('CorrelationsPage', () => {
uid: '2',
label: 'Prometheus to Loki',
config: { field: 'label', target: {}, type: 'query' },
provisioned: false,
},
]
);
@@ -580,6 +582,7 @@ describe('CorrelationsPage', () => {
targetUID: 'loki',
uid: '1',
label: 'Loki to Loki',
provisioned: false,
config: {
field: 'line',
target: {},
@@ -594,6 +597,7 @@ describe('CorrelationsPage', () => {
targetUID: 'prometheus',
uid: '2',
label: 'Loki to Prometheus',
provisioned: false,
config: {
field: 'line',
target: {},
@@ -609,6 +613,7 @@ describe('CorrelationsPage', () => {
uid: '3',
label: 'Prometheus to Loki',
config: { field: 'label', target: {}, type: 'query' },
provisioned: false,
},
{
sourceUID: 'prometheus',
@@ -616,6 +621,7 @@ describe('CorrelationsPage', () => {
uid: '4',
label: 'Prometheus to Prometheus',
config: { field: 'label', target: {}, type: 'query' },
provisioned: false,
},
]
);
@@ -638,6 +644,7 @@ describe('CorrelationsPage', () => {
targetUID: 'loki',
uid: '1',
label: 'Some label',
provisioned: true,
config: {
field: 'line',
target: {},

View File

@@ -32,7 +32,7 @@ import { CorrelationData, useCorrelations } from './useCorrelations';
const sortDatasource: SortByFn<CorrelationData> = (a, b, column) =>
a.values[column].name.localeCompare(b.values[column].name);
const isSourceReadOnly = ({ source }: Pick<CorrelationData, 'source'>) => source.readOnly;
const isCorrelationsReadOnly = (correlation: CorrelationData) => correlation.provisioned;
const loaderWrapper = css`
display: flex;
@@ -91,13 +91,14 @@ export default function CorrelationsPage() {
row: {
index,
original: {
source: { uid: sourceUID, readOnly },
source: { uid: sourceUID },
provisioned,
uid,
},
},
}: CellProps<CorrelationData, void>) => {
return (
!readOnly && (
!provisioned && (
<DeleteButton
aria-label="delete correlation"
onConfirm={() =>
@@ -118,7 +119,7 @@ export default function CorrelationsPage() {
id: 'info',
cell: InfoCell,
disableGrow: true,
visible: (data) => data.some(isSourceReadOnly),
visible: (data) => data.some(isCorrelationsReadOnly),
},
{
id: 'source',
@@ -137,7 +138,7 @@ export default function CorrelationsPage() {
id: 'actions',
cell: RowActions,
disableGrow: true,
visible: (data) => canWriteCorrelations && data.some(negate(isSourceReadOnly)),
visible: (data) => canWriteCorrelations && data.some(negate(isCorrelationsReadOnly)),
},
],
[RowActions, canWriteCorrelations]
@@ -195,7 +196,7 @@ export default function CorrelationsPage() {
<ExpendedRow
correlation={correlation}
onUpdated={handleUpdated}
readOnly={isSourceReadOnly({ source: correlation.source }) || !canWriteCorrelations}
readOnly={isCorrelationsReadOnly(correlation) || !canWriteCorrelations}
/>
)}
columns={columns}
@@ -275,7 +276,7 @@ const noWrap = css`
const InfoCell = memo(
function InfoCell({ ...props }: CellProps<CorrelationData, void>) {
const readOnly = props.row.original.source.readOnly;
const readOnly = props.row.original.provisioned;
if (readOnly) {
return <Badge text="Read only" color="purple" className={noWrap} />;

View File

@@ -49,10 +49,6 @@ export const ConfigureCorrelationSourceForm = () => {
name="sourceUID"
rules={{
required: { value: true, message: 'This field is required.' },
validate: {
writable: (uid: string) =>
!getDatasourceSrv().getInstanceSettings(uid)?.readOnly || "Source can't be a read-only data source.",
},
}}
render={({ field: { onChange, value } }) => (
<Field

View File

@@ -41,6 +41,7 @@ export interface Correlation {
targetUID: string;
label?: string;
description?: string;
provisioned: boolean;
config: CorrelationConfig;
}
@@ -49,5 +50,5 @@ export type GetCorrelationsParams = {
};
export type RemoveCorrelationParams = Pick<Correlation, 'sourceUID' | 'uid'>;
export type CreateCorrelationParams = Omit<Correlation, 'uid'>;
export type UpdateCorrelationParams = Omit<Correlation, 'targetUID'>;
export type CreateCorrelationParams = Omit<Correlation, 'uid' | 'provisioned'>;
export type UpdateCorrelationParams = Omit<Correlation, 'targetUID' | 'provisioned'>;

View File

@@ -110,6 +110,7 @@ function setup() {
source: loki,
target: prometheus,
config: { type: 'query', field: 'traceId', target: { expr: 'target Prometheus query' } },
provisioned: false,
},
// Test multiple correlations attached to the same field
{
@@ -118,6 +119,7 @@ function setup() {
source: loki,
target: elastic,
config: { type: 'query', field: 'traceId', target: { expr: 'target Elastic query' } },
provisioned: false,
},
{
uid: 'prometheus-to-elastic',
@@ -125,6 +127,7 @@ function setup() {
source: prometheus,
target: elastic,
config: { type: 'query', field: 'value', target: { expr: 'target Elastic query' } },
provisioned: false,
},
];