Provisioning: datasources auto deletion (#83034)

docs/sources/administration/provisioning/index.md

@@ -81,6 +81,14 @@ If the data source already exists, Grafana reconfigures it to match the provisio
 The configuration file can also list data sources to automatically delete, called `deleteDatasources`.
 Grafana deletes the data sources listed in `deleteDatasources` _before_ adding or updating those in the `datasources` list.
 
+You can configure Grafana to automatically delete provisioned data sources when they're removed from the provisioning file.
+To do so, add `prune: true` to the root of your provisioning file.
+With this configuration, Grafana also removes the provisioned data sources if you remove the provisioning file entirely.
+
+{{% admonition type="note" %}}
+The `prune` parameter is available in Grafana v11.1 and higher.
+{{% /admonition %}}
+
 ### Running multiple Grafana instances
 
 If you run multiple instances of Grafana, add a version number to each data source in the configuration and increase it when you update the configuration.
@@ -100,6 +108,10 @@ deleteDatasources:
   - name: Graphite
     orgId: 1
 
+# Mark provisioned data sources for deletion if they are no longer in a provisioning file.
+# It takes no effect if data sources are already listed in the deleteDatasources section.
+prune: true
+
 # List of data sources to insert/update depending on what's
 # available in the database.
 datasources:

pkg/services/datasources/datasources.go

@@ -21,6 +21,9 @@ type DataSourceService interface {
 	// GetAllDataSources gets all datasources.
 	GetAllDataSources(ctx context.Context, query *GetAllDataSourcesQuery) (res []*DataSource, err error)
 
+	// GetPrunableProvisionedDataSources gets all provisioned data sources that can be pruned.
+	GetPrunableProvisionedDataSources(ctx context.Context) (res []*DataSource, err error)
+
 	// GetDataSourcesByType gets datasources by type.
 	GetDataSourcesByType(ctx context.Context, query *GetDataSourcesByTypeQuery) ([]*DataSource, error)
 

pkg/services/datasources/fakes/fake_datasource_service.go

@@ -45,6 +45,16 @@ func (s *FakeDataSourceService) GetAllDataSources(ctx context.Context, query *da
 	return s.DataSources, nil
 }
 
+func (s *FakeDataSourceService) GetPrunableProvisionedDataSources(ctx context.Context) (res []*datasources.DataSource, err error) {
+	var dataSources []*datasources.DataSource
+	for _, dataSource := range s.DataSources {
+		if dataSource.IsPrunable {
+			dataSources = append(dataSources, dataSource)
+		}
+	}
+	return dataSources, nil
+}
+
 func (s *FakeDataSourceService) GetDataSourcesByType(ctx context.Context, query *datasources.GetDataSourcesByTypeQuery) ([]*datasources.DataSource, error) {
 	var dataSources []*datasources.DataSource
 	for _, datasource := range s.DataSources {

pkg/services/datasources/models.go

@@ -62,6 +62,8 @@ type DataSource struct {
 	SecureJsonData    map[string][]byte `json:"secureJsonData"`
 	ReadOnly          bool              `json:"readOnly"`
 	UID               string            `json:"uid" xorm:"uid"`
+	// swagger:ignore
+	IsPrunable bool `xorm:"is_prunable"`
 
 	Created time.Time `json:"created,omitempty"`
 	Updated time.Time `json:"updated,omitempty"`
@@ -161,6 +163,8 @@ type AddDataSourceCommand struct {
 	JsonData        *simplejson.Json  `json:"jsonData"`
 	SecureJsonData  map[string]string `json:"secureJsonData"`
 	UID             string            `json:"uid"`
+	// swagger:ignore
+	IsPrunable bool
 
 	OrgID                   int64             `json:"-"`
 	UserID                  int64             `json:"-"`
@@ -185,6 +189,8 @@ type UpdateDataSourceCommand struct {
 	SecureJsonData  map[string]string `json:"secureJsonData"`
 	Version         int               `json:"version"`
 	UID             string            `json:"uid"`
+	// swagger:ignore
+	IsPrunable bool
 
 	OrgID                   int64             `json:"-"`
 	ID                      int64             `json:"-"`

pkg/services/datasources/service/datasource.go

@@ -174,6 +174,10 @@ func (s *Service) GetAllDataSources(ctx context.Context, query *datasources.GetA
 	return s.SQLStore.GetAllDataSources(ctx, query)
 }
 
+func (s *Service) GetPrunableProvisionedDataSources(ctx context.Context) (res []*datasources.DataSource, err error) {
+	return s.SQLStore.GetPrunableProvisionedDataSources(ctx)
+}
+
 func (s *Service) GetDataSourcesByType(ctx context.Context, query *datasources.GetDataSourcesByTypeQuery) ([]*datasources.DataSource, error) {
 	if query.AliasIDs == nil {
 		// Populate alias IDs from plugin store

pkg/services/datasources/service/store.go

@@ -30,6 +30,7 @@ type Store interface {
 	AddDataSource(context.Context, *datasources.AddDataSourceCommand) (*datasources.DataSource, error)
 	UpdateDataSource(context.Context, *datasources.UpdateDataSourceCommand) (*datasources.DataSource, error)
 	GetAllDataSources(ctx context.Context, query *datasources.GetAllDataSourcesQuery) (res []*datasources.DataSource, err error)
+	GetPrunableProvisionedDataSources(ctx context.Context) (res []*datasources.DataSource, err error)
 
 	Count(context.Context, *quota.ScopeParameters) (*quota.Map, error)
 }
@@ -124,6 +125,16 @@ func (ss *SqlStore) GetDataSourcesByType(ctx context.Context, query *datasources
 	})
 }
 
+// GetPrunableProvisionedDataSources returns all data sources that can be pruned
+func (ss *SqlStore) GetPrunableProvisionedDataSources(ctx context.Context) ([]*datasources.DataSource, error) {
+	prunableQuery := "is_prunable  = ?"
+
+	dataSources := make([]*datasources.DataSource, 0)
+	return dataSources, ss.db.WithDbSession(ctx, func(sess *db.Session) error {
+		return sess.Where(prunableQuery, ss.db.GetDialect().BooleanStr(true)).Asc("id").Find(&dataSources)
+	})
+}
+
 // DeleteDataSource removes a datasource by org_id as well as either uid (preferred), id, or name
 // and is added to the bus. It also removes permissions related to the datasource.
 func (ss *SqlStore) DeleteDataSource(ctx context.Context, cmd *datasources.DeleteDataSourceCommand) error {
@@ -261,6 +272,7 @@ func (ss *SqlStore) AddDataSource(ctx context.Context, cmd *datasources.AddDataS
 			Version:         1,
 			ReadOnly:        cmd.ReadOnly,
 			UID:             cmd.UID,
+			IsPrunable:      cmd.IsPrunable,
 		}
 
 		if _, err := sess.Insert(ds); err != nil {
@@ -328,12 +340,14 @@ func (ss *SqlStore) UpdateDataSource(ctx context.Context, cmd *datasources.Updat
 			ReadOnly:        cmd.ReadOnly,
 			Version:         cmd.Version + 1,
 			UID:             cmd.UID,
+			IsPrunable:      cmd.IsPrunable,
 		}
 
 		sess.UseBool("is_default")
 		sess.UseBool("basic_auth")
 		sess.UseBool("with_credentials")
 		sess.UseBool("read_only")
+		sess.UseBool("is_prunable")
 		// Make sure database field is zeroed out if empty. We want to migrate away from this field.
 		sess.MustCols("database")
 		// Make sure password are zeroed out if empty. We do this as we want to migrate passwords from

pkg/services/datasources/service/store_test.go

@@ -489,5 +489,41 @@ func TestIntegrationDataAccess(t *testing.T) {
 			require.NoError(t, err)
 			require.Equal(t, 1, len(dataSources))
 		})
+
+		t.Run("Get prunable data sources", func(t *testing.T) {
+			db := db.InitTestDB(t)
+			ss := SqlStore{db: db}
+
+			_, errPrunable := ss.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
+				OrgID:      10,
+				Name:       "ElasticsearchPrunable",
+				Type:       "other",
+				Access:     datasources.DS_ACCESS_DIRECT,
+				URL:        "http://test",
+				Database:   "site",
+				ReadOnly:   true,
+				IsPrunable: true,
+			})
+			require.NoError(t, errPrunable)
+
+			_, errNotPrunable := ss.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{
+				OrgID:    10,
+				Name:     "ElasticsearchNotPrunable",
+				Type:     "other",
+				Access:   datasources.DS_ACCESS_DIRECT,
+				URL:      "http://test",
+				Database: "site",
+				ReadOnly: true,
+			})
+			require.NoError(t, errNotPrunable)
+
+			dataSources, err := ss.GetPrunableProvisionedDataSources(context.Background())
+
+			require.NoError(t, err)
+			require.Equal(t, 1, len(dataSources))
+
+			dataSource := dataSources[0]
+			require.Equal(t, "ElasticsearchPrunable", dataSource.Name)
+		})
 	})
 }

pkg/services/provisioning/datasources/config_reader_test.go

@@ -28,6 +28,8 @@ var (
 	multipleOrgsWithDefault         = "testdata/multiple-org-default"
 	withoutDefaults                 = "testdata/appliedDefaults"
 	invalidAccess                   = "testdata/invalid-access"
+	beforeAutoDeletion              = "testdata/before-auto-deletion"
+	afterAutoDeletion               = "testdata/after-auto-deletion"
 
 	oneDatasourceWithTwoCorrelations   = "testdata/one-datasource-two-correlations"
 	correlationsDifferentOrganizations = "testdata/correlations-different-organizations"
@@ -169,6 +171,39 @@ func TestDatasourceAsConfig(t *testing.T) {
 		require.Equal(t, len(store.updated), 1)
 	})
 
+	t.Run("Delete data sources when removing them from provision files", func(t *testing.T) {
+		store := &spyStore{}
+		orgFake := &orgtest.FakeOrgService{}
+		correlationsStore := &mockCorrelationsStore{}
+		dc := newDatasourceProvisioner(logger, store, correlationsStore, orgFake)
+		if err := dc.applyChanges(context.Background(), beforeAutoDeletion); err != nil {
+			t.Fatalf("applyChanges return an error %v", err)
+		}
+
+		require.Equal(t, len(store.deleted), 0)
+		require.Equal(t, len(store.inserted), 5)
+		require.Equal(t, len(store.updated), 0)
+
+		if err := dc.applyChanges(context.Background(), afterAutoDeletion); err != nil {
+			t.Fatalf("applyChanges return an error %v", err)
+		}
+
+		require.Equal(t, len(store.deleted), 2)
+
+		remainingDataSourceNames := make([]string, 3)
+
+		for i, ds := range store.items {
+			remainingDataSourceNames[i] = ds.Name
+		}
+
+		require.Contains(t, remainingDataSourceNames, "test_graphite_without_prune")
+		require.Contains(t, remainingDataSourceNames, "test_prometheus_without_prune")
+		require.Contains(t, remainingDataSourceNames, "test_graphite_with_prune")
+
+		require.NotContains(t, remainingDataSourceNames, "testdata_with_prune")
+		require.NotContains(t, remainingDataSourceNames, "test_prometheus_with_prune")
+	})
+
 	t.Run("broken yaml should return error", func(t *testing.T) {
 		reader := &configReader{}
 		_, err := reader.readConfig(context.Background(), brokenYaml)
@@ -429,6 +464,16 @@ func (s *spyStore) GetDataSource(ctx context.Context, query *datasources.GetData
 	return nil, datasources.ErrDataSourceNotFound
 }
 
+func (s *spyStore) GetPrunableProvisionedDataSources(ctx context.Context) ([]*datasources.DataSource, error) {
+	prunableProvisionedDataSources := []*datasources.DataSource{}
+	for _, item := range s.items {
+		if item.IsPrunable {
+			prunableProvisionedDataSources = append(prunableProvisionedDataSources, item)
+		}
+	}
+	return prunableProvisionedDataSources, nil
+}
+
 func (s *spyStore) DeleteDataSource(ctx context.Context, cmd *datasources.DeleteDataSourceCommand) error {
 	s.deleted = append(s.deleted, cmd)
 	for i, v := range s.items {
@@ -443,7 +488,7 @@ func (s *spyStore) DeleteDataSource(ctx context.Context, cmd *datasources.Delete
 
 func (s *spyStore) AddDataSource(ctx context.Context, cmd *datasources.AddDataSourceCommand) (*datasources.DataSource, error) {
 	s.inserted = append(s.inserted, cmd)
-	newDataSource := &datasources.DataSource{UID: cmd.UID, Name: cmd.Name, OrgID: cmd.OrgID}
+	newDataSource := &datasources.DataSource{UID: cmd.UID, Name: cmd.Name, OrgID: cmd.OrgID, IsPrunable: cmd.IsPrunable}
 	s.items = append(s.items, newDataSource)
 	return newDataSource, nil
 }

pkg/services/provisioning/datasources/datasources.go

@@ -14,6 +14,7 @@ import (
 
 type Store interface {
 	GetDataSource(ctx context.Context, query *datasources.GetDataSourceQuery) (*datasources.DataSource, error)
+	GetPrunableProvisionedDataSources(ctx context.Context) ([]*datasources.DataSource, error)
 	AddDataSource(ctx context.Context, cmd *datasources.AddDataSourceCommand) (*datasources.DataSource, error)
 	UpdateDataSource(ctx context.Context, cmd *datasources.UpdateDataSourceCommand) (*datasources.DataSource, error)
 	DeleteDataSource(ctx context.Context, cmd *datasources.DeleteDataSourceCommand) error
@@ -153,6 +154,28 @@ func (dc *DatasourceProvisioner) applyChanges(ctx context.Context, configPath st
 		}
 	}
 
+	prunableProvisionedDataSources, err := dc.store.GetPrunableProvisionedDataSources(ctx)
+	if err != nil {
+		return err
+	}
+
+	staleProvisionedDataSources := []*deleteDatasourceConfig{}
+
+	for _, prunableProvisionedDataSource := range prunableProvisionedDataSources {
+		key := DataSourceMapKey{
+			OrgId: prunableProvisionedDataSource.OrgID,
+			Name:  prunableProvisionedDataSource.Name,
+		}
+		if _, ok := willExistAfterProvisioning[key]; !ok {
+			staleProvisionedDataSources = append(staleProvisionedDataSources, &deleteDatasourceConfig{OrgID: prunableProvisionedDataSource.OrgID, Name: prunableProvisionedDataSource.Name})
+			willExistAfterProvisioning[key] = false
+		}
+	}
+
+	if err := dc.deleteDatasources(ctx, staleProvisionedDataSources, willExistAfterProvisioning); err != nil {
+		return err
+	}
+
 	for _, cfg := range configs {
 		if err := dc.provisionDataSources(ctx, cfg, willExistAfterProvisioning); err != nil {
 			return err

pkg/services/provisioning/datasources/testdata/after-auto-deletion/provision1.yaml

@@ -0,0 +1,6 @@
+apiVersion: 1
+datasources:
+  - name: test_graphite_without_prune
+    type: graphite
+    access: proxy
+    url: http://localhost:8080

pkg/services/provisioning/datasources/testdata/after-auto-deletion/provision3.yaml

@@ -0,0 +1,7 @@
+apiVersion: 1
+prune: true
+datasources:
+  - name: test_graphite_with_prune
+    type: graphite
+    access: proxy
+    url: http://localhost:8080

pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision1.yaml

@@ -0,0 +1,10 @@
+apiVersion: 1
+datasources:
+  - name: test_graphite_without_prune
+    type: graphite
+    access: proxy
+    url: http://localhost:8080
+  - name: test_prometheus_without_prune
+    type: prometheus
+    access: proxy
+    url: http://localhost:9090

pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision2.yaml

@@ -0,0 +1,5 @@
+apiVersion: 1
+prune: true
+datasources:
+  - name: testdata_with_prune
+    type: testdata

pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision3.yaml

@@ -0,0 +1,11 @@
+apiVersion: 1
+prune: true
+datasources:
+  - name: test_graphite_with_prune
+    type: graphite
+    access: proxy
+    url: http://localhost:8080
+  - name: test_prometheus_with_prune
+    type: prometheus
+    access: proxy
+    url: http://localhost:9090

pkg/services/provisioning/datasources/types.go

@@ -18,6 +18,7 @@ type configVersion struct {
 
 type configs struct {
 	APIVersion int64
+	Prune      bool
 
 	Datasources       []*upsertDataSourceFromConfig
 	DeleteDatasources []*deleteDatasourceConfig
@@ -47,6 +48,7 @@ type upsertDataSourceFromConfig struct {
 	SecureJSONData  map[string]string
 	Editable        bool
 	UID             string
+	IsPrunable      bool
 }
 
 type configsV0 struct {
@@ -58,7 +60,8 @@ type configsV0 struct {
 
 type configsV1 struct {
 	configVersion
-	log log.Logger
+	log   log.Logger
+	Prune bool
 
 	Datasources       []*upsertDataSourceFromConfigV1 `json:"datasources" yaml:"datasources"`
 	DeleteDatasources []*deleteDatasourceConfigV1     `json:"deleteDatasources" yaml:"deleteDatasources"`
@@ -111,6 +114,7 @@ type upsertDataSourceFromConfigV1 struct {
 	SecureJSONData  values.StringMapValue `json:"secureJsonData" yaml:"secureJsonData"`
 	Editable        values.BoolValue      `json:"editable" yaml:"editable"`
 	UID             values.StringValue    `json:"uid" yaml:"uid"`
+	IsPrunable      values.BoolValue
 }
 
 func (cfg *configsV1) mapToDatasourceFromConfig(apiVersion int64) *configs {
@@ -141,6 +145,7 @@ func (cfg *configsV1) mapToDatasourceFromConfig(apiVersion int64) *configs {
 			Editable:        ds.Editable.Value(),
 			Version:         ds.Version.Value(),
 			UID:             ds.UID.Value(),
+			IsPrunable:      cfg.Prune,
 		})
 	}
 
@@ -218,6 +223,7 @@ func createInsertCommand(ds *upsertDataSourceFromConfig) *datasources.AddDataSou
 		SecureJsonData:  ds.SecureJSONData,
 		ReadOnly:        !ds.Editable,
 		UID:             ds.UID,
+		IsPrunable:      ds.IsPrunable,
 	}
 
 	if cmd.UID == "" {
@@ -260,5 +266,6 @@ func createUpdateCommand(ds *upsertDataSourceFromConfig, id int64) *datasources.
 		SecureJsonData:          ds.SecureJSONData,
 		ReadOnly:                !ds.Editable,
 		IgnoreOldSecureJsonData: true,
+		IsPrunable:              ds.IsPrunable,
 	}
 }

pkg/services/sqlstore/migrations/datasource_mig.go

@@ -134,4 +134,8 @@ func addDataSourceMigration(mg *Migrator) {
 
 	mg.AddMigration("add unique index datasource_org_id_is_default", NewAddIndexMigration(tableV2, &Index{
 		Cols: []string{"org_id", "is_default"}}))
+
+	mg.AddMigration("Add is_prunable column", NewAddColumnMigration(tableV2, &Column{
+		Name: "is_prunable", Type: DB_Bool, Nullable: true, Default: "0",
+	}))
 }