diff --git a/pkg/registry/apis/dashboard/legacy/sql_dashboards.go b/pkg/registry/apis/dashboard/legacy/sql_dashboards.go index b90f218d0aa..ec751cbecf0 100644 --- a/pkg/registry/apis/dashboard/legacy/sql_dashboards.go +++ b/pkg/registry/apis/dashboard/legacy/sql_dashboards.go @@ -22,6 +22,7 @@ import ( "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" gapiutil "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/dashboards/service" "github.com/grafana/grafana/pkg/services/provisioning" "github.com/grafana/grafana/pkg/storage/legacysql" "github.com/grafana/grafana/pkg/storage/unified/resource" @@ -278,21 +279,24 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows) (*dashboardRow, error) { if origin_name.String != "" { ts := time.Unix(origin_ts.Int64, 0) - resolvedPath := a.provisioning.GetDashboardProvisionerResolvedPath(origin_name.String) - originPath, err := filepath.Rel( - resolvedPath, - origin_path.String, - ) - if err != nil { - return nil, err - } - - meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{ + repo := &utils.ResourceRepositoryInfo{ Name: origin_name.String, - Path: originPath, Hash: origin_hash.String, Timestamp: &ts, - }) + } + // if the reader cannot be found, it may be an orphaned provisioned dashboard + resolvedPath := a.provisioning.GetDashboardProvisionerResolvedPath(origin_name.String) + if resolvedPath != "" { + originPath, err := filepath.Rel( + resolvedPath, + origin_path.String, + ) + if err != nil { + return nil, err + } + repo.Path = originPath + } + meta.SetRepositoryInfo(repo) } else if plugin_id != "" { meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{ Name: "plugin", @@ -390,6 +394,7 @@ func (a *dashboardSqlAccess) SaveDashboard(ctx context.Context, orgId int64, das } out, err := a.dashStore.SaveDashboard(ctx, dashboards.SaveDashboardCommand{ OrgID: orgId, + PluginID: service.GetPluginIDFromMeta(meta), Dashboard: simplejson.NewFromAny(dash.Spec.UnstructuredContent()), FolderUID: meta.GetFolder(), Overwrite: true, // already passed the revisionVersion checks! diff --git a/pkg/services/dashboards/dashboard.go b/pkg/services/dashboards/dashboard.go index 6931b69ffb7..ff51f66622c 100644 --- a/pkg/services/dashboards/dashboard.go +++ b/pkg/services/dashboards/dashboard.go @@ -62,6 +62,7 @@ type DashboardProvisioningService interface { //go:generate mockery --name Store --structname FakeDashboardStore --inpackage --filename store_mock.go type Store interface { DeleteDashboard(ctx context.Context, cmd *DeleteDashboardCommand) error + CleanupAfterDelete(ctx context.Context, cmd *DeleteDashboardCommand) error DeleteAllDashboards(ctx context.Context, orgID int64) error DeleteOrphanedProvisionedDashboards(ctx context.Context, cmd *DeleteOrphanedProvisionedDashboardsCommand) error FindDashboards(ctx context.Context, query *FindPersistedDashboardsQuery) ([]DashboardSearchProjection, error) @@ -75,7 +76,7 @@ type Store interface { GetProvisionedDataByDashboardID(ctx context.Context, dashboardID int64) (*DashboardProvisioning, error) GetProvisionedDataByDashboardUID(ctx context.Context, orgID int64, dashboardUID string) (*DashboardProvisioning, error) SaveDashboard(ctx context.Context, cmd SaveDashboardCommand) (*Dashboard, error) - SaveProvisionedDashboard(ctx context.Context, cmd SaveDashboardCommand, provisioning *DashboardProvisioning) (*Dashboard, error) + SaveProvisionedDashboard(ctx context.Context, dash *Dashboard, provisioning *DashboardProvisioning) error UnprovisionDashboard(ctx context.Context, id int64) error // ValidateDashboardBeforeSave validates a dashboard before save. ValidateDashboardBeforeSave(ctx context.Context, dashboard *Dashboard, overwrite bool) (bool, error) diff --git a/pkg/services/dashboards/database/database.go b/pkg/services/dashboards/database/database.go index 132ce8b6e13..5c19adb9427 100644 --- a/pkg/services/dashboards/database/database.go +++ b/pkg/services/dashboards/database/database.go @@ -145,25 +145,19 @@ func (d *dashboardStore) GetProvisionedDashboardData(ctx context.Context, name s return result, err } -func (d *dashboardStore) SaveProvisionedDashboard(ctx context.Context, cmd dashboards.SaveDashboardCommand, provisioning *dashboards.DashboardProvisioning) (*dashboards.Dashboard, error) { +func (d *dashboardStore) SaveProvisionedDashboard(ctx context.Context, dash *dashboards.Dashboard, provisioning *dashboards.DashboardProvisioning) error { ctx, span := tracer.Start(ctx, "dashboards.database.SaveProvisionedDashboard") defer span.End() - var result *dashboards.Dashboard - var err error - err = d.store.WithTransactionalDbSession(ctx, func(sess *db.Session) error { - result, err = saveDashboard(sess, &cmd, d.emitEntityEvent()) - if err != nil { - return err - } - + err := d.store.WithTransactionalDbSession(ctx, func(sess *db.Session) error { if provisioning.Updated == 0 { - provisioning.Updated = result.Updated.Unix() + provisioning.Updated = dash.Updated.Unix() } - return saveProvisionedData(sess, provisioning, result) + return saveProvisionedData(sess, provisioning, dash) }) - return result, err + + return err } func (d *dashboardStore) SaveDashboard(ctx context.Context, cmd dashboards.SaveDashboardCommand) (*dashboards.Dashboard, error) { @@ -656,6 +650,45 @@ func (d *dashboardStore) deleteDashboard(cmd *dashboards.DeleteDashboardCommand, return nil } +func (d *dashboardStore) CleanupAfterDelete(ctx context.Context, cmd *dashboards.DeleteDashboardCommand) error { + type statement struct { + SQL string + args []any + } + sqlStatements := []statement{ + {SQL: "DELETE FROM dashboard_tag WHERE dashboard_uid = ? AND org_id = ?", args: []any{cmd.UID, cmd.OrgID}}, + {SQL: "DELETE FROM star WHERE dashboard_uid = ? AND org_id = ?", args: []any{cmd.UID, cmd.OrgID}}, + {SQL: "DELETE FROM playlist_item WHERE type = 'dashboard_by_id' AND value = ?", args: []any{cmd.ID}}, + {SQL: "DELETE FROM dashboard_version WHERE dashboard_id = ?", args: []any{cmd.ID}}, + {SQL: "DELETE FROM dashboard_provisioning WHERE dashboard_id = ?", args: []any{cmd.ID}}, + {SQL: "DELETE FROM dashboard_acl WHERE dashboard_id = ?", args: []any{cmd.ID}}, + {SQL: "DELETE FROM annotation WHERE dashboard_id = ? AND org_id = ?", args: []any{cmd.ID, cmd.OrgID}}, + } + + err := d.store.WithTransactionalDbSession(ctx, func(sess *db.Session) error { + err := d.store.WithDbSession(ctx, func(sess *db.Session) error { + for _, stmnt := range sqlStatements { + _, err := sess.Exec(append([]any{stmnt.SQL}, stmnt.args...)...) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + return err + } + + if err := d.deleteResourcePermissions(sess, cmd.OrgID, ac.GetResourceScopeUID("dashboards", cmd.UID)); err != nil { + return err + } + + return nil + }) + + return err +} + func (d *dashboardStore) DeleteAllDashboards(ctx context.Context, orgID int64) error { ctx, span := tracer.Start(ctx, "dashboards.database.DeleteAllDashboards") defer span.End() diff --git a/pkg/services/dashboards/database/database_provisioning_test.go b/pkg/services/dashboards/database/database_provisioning_test.go index a10af1f3d19..1cf53ce5fb2 100644 --- a/pkg/services/dashboards/database/database_provisioning_test.go +++ b/pkg/services/dashboards/database/database_provisioning_test.go @@ -52,13 +52,15 @@ func TestIntegrationDashboardProvisioningTest(t *testing.T) { ExternalID: "/var/grafana.json", Updated: now.Unix(), } - - dash, err := dashboardStore.SaveProvisionedDashboard(context.Background(), saveDashboardCmd, provisioning) - require.Nil(t, err) + dash, err := dashboardStore.SaveDashboard(context.Background(), saveDashboardCmd) + require.NoError(t, err) require.NotNil(t, dash) require.NotEqual(t, 0, dash.ID) dashId := dash.ID + err = dashboardStore.SaveProvisionedDashboard(context.Background(), dash, provisioning) + require.Nil(t, err) + t.Run("Deleting orphaned provisioned dashboards", func(t *testing.T) { saveCmd := dashboards.SaveDashboardCommand{ OrgID: 1, @@ -69,13 +71,16 @@ func TestIntegrationDashboardProvisioningTest(t *testing.T) { "title": "another_dashboard", }), } + anotherDash, err := dashboardStore.SaveDashboard(context.Background(), saveCmd) + require.NoError(t, err) + provisioning := &dashboards.DashboardProvisioning{ Name: "another_reader", ExternalID: "/var/grafana.json", Updated: now.Unix(), } - anotherDash, err := dashboardStore.SaveProvisionedDashboard(context.Background(), saveCmd, provisioning) + err = dashboardStore.SaveProvisionedDashboard(context.Background(), anotherDash, provisioning) require.Nil(t, err) query := &dashboards.GetDashboardsQuery{DashboardIDs: []int64{anotherDash.ID}} diff --git a/pkg/services/dashboards/database/database_test.go b/pkg/services/dashboards/database/database_test.go index d393d9a3488..9628d111b56 100644 --- a/pkg/services/dashboards/database/database_test.go +++ b/pkg/services/dashboards/database/database_test.go @@ -238,6 +238,38 @@ func TestIntegrationDashboardDataAccess(t *testing.T) { require.NotContains(t, terms, "delete this") }) + t.Run("Should delete associated provisioning info, even without the dashboard existing in the db", func(t *testing.T) { + setup() + provisioningData := &dashboards.DashboardProvisioning{ + ID: 1, + DashboardID: 200, + Name: "test", + CheckSum: "123", + Updated: 54321, + ExternalID: "/path/to/dashboard", + } + err := dashboardStore.SaveProvisionedDashboard(context.Background(), &dashboards.Dashboard{ + ID: 200, + }, provisioningData) + require.NoError(t, err) + + res, err := dashboardStore.GetProvisionedDashboardData(context.Background(), "test") + require.NoError(t, err) + require.Len(t, res, 1) + require.Equal(t, res[0], provisioningData) + + err = dashboardStore.CleanupAfterDelete(context.Background(), &dashboards.DeleteDashboardCommand{ + ID: 200, + OrgID: 1, + UID: "test", + }) + require.NoError(t, err) + + res, err = dashboardStore.GetProvisionedDashboardData(context.Background(), "test") + require.NoError(t, err) + require.Len(t, res, 0) + }) + t.Run("Should be able to delete all dashboards for an org", func(t *testing.T) { setup() dash1 := insertTestDashboard(t, dashboardStore, "delete me", 1, 0, "", false, "delete this") diff --git a/pkg/services/dashboards/models.go b/pkg/services/dashboards/models.go index be1d8032407..ee52b111605 100644 --- a/pkg/services/dashboards/models.go +++ b/pkg/services/dashboards/models.go @@ -429,6 +429,10 @@ type FindPersistedDashboardsQuery struct { Sort model.SortOption IsDeleted bool + ProvisionedRepo string + ProvisionedPath string + ProvisionedReposNotIn []string + Filters []any // Skip access control checks. This field is used by OpenFGA search implementation. diff --git a/pkg/services/dashboards/service/dashboard_service.go b/pkg/services/dashboards/service/dashboard_service.go index 8c506d2965e..692d6d54383 100644 --- a/pkg/services/dashboards/service/dashboard_service.go +++ b/pkg/services/dashboards/service/dashboard_service.go @@ -8,15 +8,19 @@ import ( "fmt" "strconv" "strings" + "sync" "time" "github.com/google/uuid" "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel" "golang.org/x/exp/slices" + "golang.org/x/sync/errgroup" + apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/selection" k8sUser "k8s.io/apiserver/pkg/authentication/user" k8sRequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/client-go/dynamic" @@ -165,7 +169,7 @@ func (dr *DashboardServiceImpl) Count(ctx context.Context, scopeParams *quota.Sc total := int64(0) for _, org := range orgs { - ctx = identity.WithRequester(ctx, getQuotaRequester(org.ID)) + ctx = identity.WithRequester(ctx, getDashboardBackgroundRequester(org.ID)) dashs, err := dr.listDashboardsThroughK8s(ctx, org.ID) if err != nil { return u, err @@ -213,13 +217,13 @@ func readQuotaConfig(cfg *setting.Cfg) (*quota.Map, error) { return limits, nil } -func getQuotaRequester(orgId int64) *identity.StaticRequester { +func getDashboardBackgroundRequester(orgId int64) *identity.StaticRequester { return &identity.StaticRequester{ Type: claims.TypeServiceAccount, UserID: 1, OrgID: orgId, - Name: "quota-requester", - Login: "quota-requester", + Name: "dashboard-background", + Login: "dashboard-background", Permissions: map[int64]map[string][]string{ orgId: { "*": {"*"}, @@ -229,15 +233,103 @@ func getQuotaRequester(orgId int64) *identity.StaticRequester { } func (dr *DashboardServiceImpl) GetProvisionedDashboardData(ctx context.Context, name string) ([]*dashboards.DashboardProvisioning, error) { + if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) { + orgs, err := dr.orgService.Search(ctx, &org.SearchOrgsQuery{}) + if err != nil { + return nil, err + } + + results := []*dashboards.DashboardProvisioning{} + var mu sync.Mutex + g, ctx := errgroup.WithContext(ctx) + for _, org := range orgs { + func(orgID int64) { + g.Go(func() error { + res, err := dr.searchProvisionedDashboardsThroughK8s(ctx, dashboards.FindPersistedDashboardsQuery{ + ProvisionedRepo: name, + OrgId: orgID, + }) + if err != nil { + return err + } + + mu.Lock() + for _, r := range res { + results = append(results, &r.DashboardProvisioning) + } + mu.Unlock() + return nil + }) + }(org.ID) + } + + if err := g.Wait(); err != nil { + return nil, err + } + + return results, nil + } + return dr.dashboardStore.GetProvisionedDashboardData(ctx, name) } func (dr *DashboardServiceImpl) GetProvisionedDashboardDataByDashboardID(ctx context.Context, dashboardID int64) (*dashboards.DashboardProvisioning, error) { + if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) { + // if dashboard id is 0, it is a new dashboard + if dashboardID == 0 { + return nil, nil + } + + orgs, err := dr.orgService.Search(ctx, &org.SearchOrgsQuery{}) + if err != nil { + return nil, err + } + + for _, org := range orgs { + res, err := dr.searchProvisionedDashboardsThroughK8s(ctx, dashboards.FindPersistedDashboardsQuery{ + OrgId: org.ID, + DashboardIds: []int64{dashboardID}, + }) + if err != nil { + return nil, err + } + + if len(res) == 1 { + return &res[0].DashboardProvisioning, nil + } else if len(res) > 1 { + return nil, fmt.Errorf("found more than one provisioned dashboard with ID %d", dashboardID) + } + } + + return nil, nil + } + return dr.dashboardStore.GetProvisionedDataByDashboardID(ctx, dashboardID) } func (dr *DashboardServiceImpl) GetProvisionedDashboardDataByDashboardUID(ctx context.Context, orgID int64, dashboardUID string) (*dashboards.DashboardProvisioning, error) { - // TODO: make this go through the k8s cli too under the feature toggle. First get dashboard through unistore & then get provisioning data + if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) { + if dashboardUID == "" { + return nil, nil + } + + res, err := dr.searchProvisionedDashboardsThroughK8s(ctx, dashboards.FindPersistedDashboardsQuery{ + OrgId: orgID, + DashboardUIDs: []string{dashboardUID}, + }) + if err != nil { + return nil, err + } + + if len(res) == 1 { + return &res[0].DashboardProvisioning, nil + } else if len(res) > 1 { + return nil, fmt.Errorf("found more than one provisioned dashboard with UID %s", dashboardUID) + } + + return nil, nil + } + return dr.dashboardStore.GetProvisionedDataByDashboardUID(ctx, orgID, dashboardUID) } @@ -452,7 +544,34 @@ func (dr *DashboardServiceImpl) ValidateDashboardBeforeSave(ctx context.Context, } func (dr *DashboardServiceImpl) DeleteOrphanedProvisionedDashboards(ctx context.Context, cmd *dashboards.DeleteOrphanedProvisionedDashboardsCommand) error { - // TODO: once we can search in unistore by id, go through k8s cli too + if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) { + // check each org for orphaned provisioned dashboards + orgs, err := dr.orgService.Search(ctx, &org.SearchOrgsQuery{}) + if err != nil { + return err + } + + for _, org := range orgs { + // find all dashboards in the org that have a file repo set that is not in the given readers list + foundDashs, err := dr.searchProvisionedDashboardsThroughK8s(ctx, dashboards.FindPersistedDashboardsQuery{ + ProvisionedReposNotIn: cmd.ReaderNames, + OrgId: org.ID, + }) + if err != nil { + return err + } + + // delete them + for _, foundDash := range foundDashs { + ctx = identity.WithRequester(ctx, getDashboardBackgroundRequester(org.ID)) + if err = dr.deleteDashboard(ctx, foundDash.DashboardID, foundDash.DashboardUID, org.ID, false); err != nil { + return err + } + } + } + return nil + } + return dr.dashboardStore.DeleteOrphanedProvisionedDashboards(ctx, cmd) } @@ -520,15 +639,30 @@ func (dr *DashboardServiceImpl) SaveProvisionedDashboard(ctx context.Context, dt } dto.User = accesscontrol.BackgroundUser("dashboard_provisioning", dto.OrgID, org.RoleAdmin, provisionerPermissions) + ctx = identity.WithRequester(ctx, getDashboardBackgroundRequester(dto.OrgID)) cmd, err := dr.BuildSaveDashboardCommand(ctx, dto, false) if err != nil { return nil, err } - // dashboard - // TODO: make this go through the k8s cli too under the feature toggle. First save dashboard & then save provisioning data - dash, err := dr.dashboardStore.SaveProvisionedDashboard(ctx, *cmd, provisioning) + var dash *dashboards.Dashboard + if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) { + // save the dashboard but then do NOT return + // we want to save the provisioning data to the dashboard_provisioning table still + // to ensure we can safely rollback to mode2 if needed + dash, err = dr.saveProvisionedDashboardThroughK8s(ctx, cmd, provisioning, false) + if err != nil { + return nil, err + } + } else { + dash, err = dr.saveDashboard(ctx, cmd) + if err != nil { + return nil, err + } + } + + err = dr.dashboardStore.SaveProvisionedDashboard(ctx, dash, provisioning) if err != nil { return nil, err } @@ -545,6 +679,8 @@ func (dr *DashboardServiceImpl) SaveFolderForProvisionedDashboards(ctx context.C defer span.End() dto.SignedInUser = accesscontrol.BackgroundUser("dashboard_provisioning", dto.OrgID, org.RoleAdmin, provisionerPermissions) + ctx = identity.WithRequester(ctx, getDashboardBackgroundRequester(dto.OrgID)) + f, err := dr.folderService.Create(ctx, dto) if err != nil { dr.log.Error("failed to create folder for provisioned dashboards", "folder", dto.Title, "org", dto.OrgID, "err", err) @@ -687,6 +823,7 @@ func (dr *DashboardServiceImpl) GetDashboardByPublicUid(ctx context.Context, das // DeleteProvisionedDashboard removes dashboard from the DB even if it is provisioned. func (dr *DashboardServiceImpl) DeleteProvisionedDashboard(ctx context.Context, dashboardId int64, orgId int64) error { + ctx = identity.WithRequester(ctx, getDashboardBackgroundRequester(orgId)) return dr.deleteDashboard(ctx, dashboardId, "", orgId, false) } @@ -697,7 +834,12 @@ func (dr *DashboardServiceImpl) deleteDashboard(ctx context.Context, dashboardId cmd := &dashboards.DeleteDashboardCommand{OrgID: orgId, ID: dashboardId, UID: dashboardUID} if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) { - return dr.deleteDashboardThroughK8s(ctx, cmd, validateProvisionedDashboard) + err := dr.deleteDashboardThroughK8s(ctx, cmd, validateProvisionedDashboard) + if err != nil { + return err + } + // cleanup things related to dashboards that are not stored in unistore yet + return dr.dashboardStore.CleanupAfterDelete(ctx, cmd) } if validateProvisionedDashboard { @@ -744,12 +886,46 @@ func (dr *DashboardServiceImpl) ImportDashboard(ctx context.Context, dto *dashbo // UnprovisionDashboard removes info about dashboard being provisioned. Used after provisioning configs are changed // and provisioned dashboards are left behind but not deleted. func (dr *DashboardServiceImpl) UnprovisionDashboard(ctx context.Context, dashboardId int64) error { - // TODO: once we can search by dashboard ID in unistore, go through k8s cli too + if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) { + orgs, err := dr.orgService.Search(ctx, &org.SearchOrgsQuery{}) + if err != nil { + return err + } + + for _, org := range orgs { + ctx = identity.WithRequester(ctx, getDashboardBackgroundRequester(org.ID)) + dash, err := dr.getDashboardThroughK8s(ctx, &dashboards.GetDashboardQuery{OrgID: org.ID, ID: dashboardId}) + if err != nil { + // if we can't find it in this org, try the next one + continue + } + + _, err = dr.saveProvisionedDashboardThroughK8s(ctx, &dashboards.SaveDashboardCommand{ + OrgID: org.ID, + PluginID: dash.PluginID, + FolderUID: dash.FolderUID, + FolderID: dash.FolderID, // nolint:staticcheck + UpdatedAt: time.Now(), + Dashboard: dash.Data, + }, nil, true) + + return err + } + + return dashboards.ErrDashboardNotFound + } + return dr.dashboardStore.UnprovisionDashboard(ctx, dashboardId) } func (dr *DashboardServiceImpl) GetDashboardsByPluginID(ctx context.Context, query *dashboards.GetDashboardsByPluginIDQuery) ([]*dashboards.Dashboard, error) { - // TODO: once we can do this search in unistore, go through k8s cli too + if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) { + return dr.searchDashboardsThroughK8s(ctx, &dashboards.FindPersistedDashboardsQuery{ + OrgId: query.OrgID, + ProvisionedRepo: pluginIDRepoName, + ProvisionedPath: query.PluginID, + }) + } return dr.dashboardStore.GetDashboardsByPluginID(ctx, query) } @@ -1220,10 +1396,16 @@ func (dr *DashboardServiceImpl) CleanUpDeletedDashboards(ctx context.Context) (i // ----------------------------------------------------------------------------------------- func (dk8s *dashk8sHandler) getClient(ctx context.Context, orgID int64) (dynamic.ResourceInterface, bool) { - dyn, err := dynamic.NewForConfig(dk8s.restConfigProvider.GetRestConfig(ctx)) + cfg := dk8s.restConfigProvider.GetRestConfig(ctx) + if cfg == nil { + return nil, false + } + + dyn, err := dynamic.NewForConfig(cfg) if err != nil { return nil, false } + return dyn.Resource(dk8s.gvr).Namespace(dk8s.getNamespace(orgID)), true } @@ -1305,15 +1487,81 @@ func (dr *DashboardServiceImpl) getDashboardThroughK8s(ctx context.Context, quer } out, err := client.Get(newCtx, query.UID, v1.GetOptions{}, subresource) - if err != nil { + if err != nil && !apierrors.IsNotFound(err) { return nil, err - } else if out == nil { + } else if err != nil || out == nil { return nil, dashboards.ErrDashboardNotFound } return dr.UnstructuredToLegacyDashboard(ctx, out, query.OrgID) } +func (dr *DashboardServiceImpl) saveProvisionedDashboardThroughK8s(ctx context.Context, cmd *dashboards.SaveDashboardCommand, provisioning *dashboards.DashboardProvisioning, unprovision bool) (*dashboards.Dashboard, error) { + // default to 1 if not set + if cmd.OrgID == 0 { + cmd.OrgID = 1 + } + + // create a new context - prevents issues when the request stems from the k8s api itself + // otherwise the context goes through the handlers twice and causes issues + newCtx, cancel, err := dr.getK8sContext(ctx) + if err != nil { + return nil, err + } else if cancel != nil { + defer cancel() + } + + client, ok := dr.k8sclient.getClient(newCtx, cmd.OrgID) + if !ok { + return nil, nil + } + + obj, err := LegacySaveCommandToUnstructured(cmd, dr.k8sclient.getNamespace(cmd.OrgID)) + if err != nil { + return nil, err + } + + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + if unprovision { + delete(annotations, utils.AnnoKeyRepoName) + delete(annotations, utils.AnnoKeyRepoPath) + delete(annotations, utils.AnnoKeyRepoHash) + delete(annotations, utils.AnnoKeyRepoTimestamp) + } else { + annotations[utils.AnnoKeyRepoName] = provisionedFileNameWithPrefix(provisioning.Name) + annotations[utils.AnnoKeyRepoPath] = provisioning.ExternalID + annotations[utils.AnnoKeyRepoHash] = provisioning.CheckSum + annotations[utils.AnnoKeyRepoTimestamp] = time.Unix(provisioning.Updated, 0).UTC().Format(time.RFC3339) + } + obj.SetAnnotations(annotations) + + var out *unstructured.Unstructured + current, err := client.Get(newCtx, obj.GetName(), v1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return nil, err + } else if current == nil || (err != nil && apierrors.IsNotFound(err)) { + out, err = client.Create(newCtx, &obj, v1.CreateOptions{}) + if err != nil { + return nil, err + } + } else { + out, err = client.Update(newCtx, &obj, v1.UpdateOptions{}) + if err != nil { + return nil, err + } + } + + finalDash, err := dr.UnstructuredToLegacyDashboard(ctx, out, cmd.OrgID) + if err != nil { + return nil, err + } + + return finalDash, nil +} + func (dr *DashboardServiceImpl) saveDashboardThroughK8s(ctx context.Context, cmd *dashboards.SaveDashboardCommand, orgID int64) (*dashboards.Dashboard, error) { // create a new context - prevents issues when the request stems from the k8s api itself // otherwise the context goes through the handlers twice and causes issues @@ -1334,6 +1582,8 @@ func (dr *DashboardServiceImpl) saveDashboardThroughK8s(ctx context.Context, cmd return nil, err } + setPluginID(obj, cmd.PluginID) + var out *unstructured.Unstructured current, err := client.Get(newCtx, obj.GetName(), v1.GetOptions{}) if current == nil || err != nil { @@ -1466,40 +1716,64 @@ func (dr *DashboardServiceImpl) searchDashboardsThroughK8sRaw(ctx context.Contex request := &resource.ResourceSearchRequest{ Options: &resource.ListOptions{ - Key: dashboardskey, + Key: dashboardskey, + Fields: []*resource.Requirement{}, + Labels: []*resource.Requirement{}, }, Limit: 100000} if len(query.DashboardUIDs) > 0 { request.Options.Fields = []*resource.Requirement{{ Key: "key.name", - Operator: "in", + Operator: string(selection.In), Values: query.DashboardUIDs, }} } else if len(query.DashboardIds) > 0 { values := make([]string, len(query.DashboardIds)) - for _, id := range query.DashboardIds { - values = append(values, strconv.FormatInt(id, 10)) + for i, id := range query.DashboardIds { + values[i] = strconv.FormatInt(id, 10) } - request.Options.Labels = []*resource.Requirement{{ + request.Options.Labels = append(request.Options.Labels, &resource.Requirement{ Key: utils.LabelKeyDeprecatedInternalID, // nolint:staticcheck - Operator: "in", + Operator: string(selection.In), Values: values, - }} + }) } if len(query.FolderUIDs) > 0 { req := []*resource.Requirement{{ Key: "folder", - Operator: "in", + Operator: string(selection.In), Values: query.FolderUIDs, }} - if len(request.Options.Fields) == 0 { - request.Options.Fields = req - } else { - request.Options.Fields = append(request.Options.Fields, req...) - } + request.Options.Fields = append(request.Options.Fields, req...) + } + + if query.ProvisionedRepo != "" { + req := []*resource.Requirement{{ + Key: "repo.name", + Operator: string(selection.In), + Values: []string{query.ProvisionedRepo}, + }} + request.Options.Fields = append(request.Options.Fields, req...) + } + + if len(query.ProvisionedReposNotIn) > 0 { + req := []*resource.Requirement{{ + Key: "repo.name", + Operator: string(selection.NotIn), + Values: query.ProvisionedReposNotIn, + }} + request.Options.Fields = append(request.Options.Fields, req...) + } + if query.ProvisionedPath != "" { + req := []*resource.Requirement{{ + Key: "repo.path", + Operator: string(selection.In), + Values: []string{query.ProvisionedPath}, + }} + request.Options.Fields = append(request.Options.Fields, req...) } // note: this does not allow for partial matching @@ -1509,28 +1783,19 @@ func (dr *DashboardServiceImpl) searchDashboardsThroughK8sRaw(ctx context.Contex if query.Title != "" { req := []*resource.Requirement{{ Key: "title", - Operator: "in", + Operator: string(selection.In), Values: []string{query.Title}, }} - if len(request.Options.Fields) == 0 { - request.Options.Fields = req - } else { - request.Options.Fields = append(request.Options.Fields, req...) - } + request.Options.Fields = append(request.Options.Fields, req...) } if len(query.Tags) > 0 { req := []*resource.Requirement{{ Key: "tags", - Operator: "in", + Operator: string(selection.In), Values: query.Tags, }} - - if len(request.Options.Fields) == 0 { - request.Options.Fields = req - } else { - request.Options.Fields = append(request.Options.Fields, req...) - } + request.Options.Fields = append(request.Options.Fields, req...) } res, err := dr.k8sclient.getSearcher().Search(ctx, request) @@ -1541,6 +1806,100 @@ func (dr *DashboardServiceImpl) searchDashboardsThroughK8sRaw(ctx context.Contex return ParseResults(res, 0) } +type dashboardProvisioningWithUID struct { + dashboards.DashboardProvisioning + DashboardUID string +} + +func (dr *DashboardServiceImpl) searchProvisionedDashboardsThroughK8s(ctx context.Context, query dashboards.FindPersistedDashboardsQuery) ([]*dashboardProvisioningWithUID, error) { + ctx = identity.WithRequester(ctx, getDashboardBackgroundRequester(query.OrgId)) + + if query.ProvisionedRepo != "" { + query.ProvisionedRepo = provisionedFileNameWithPrefix(query.ProvisionedRepo) + } + + if len(query.ProvisionedReposNotIn) > 0 { + repos := make([]string, len(query.ProvisionedReposNotIn)) + for i, v := range query.ProvisionedReposNotIn { + repos[i] = provisionedFileNameWithPrefix(v) + } + query.ProvisionedReposNotIn = repos + } + + searchResults, err := dr.searchDashboardsThroughK8sRaw(ctx, &query) + if err != nil { + return nil, err + } + + newCtx, cancel, err := dr.getK8sContext(ctx) + if err != nil { + return nil, err + } else if cancel != nil { + defer cancel() + } + + client, ok := dr.k8sclient.getClient(newCtx, query.OrgId) + if !ok { + return nil, nil + } + + // loop through all hits concurrently to get the repo information (if set due to file provisioning) + dashs := make([]*dashboardProvisioningWithUID, 0) + var mu sync.Mutex + g, ctx := errgroup.WithContext(ctx) + for _, h := range searchResults.Hits { + func(hit v0alpha1.DashboardHit) { + g.Go(func() error { + out, err := client.Get(ctx, hit.Name, v1.GetOptions{}, "") + if err != nil { + return err + } else if out == nil { + return dashboards.ErrDashboardNotFound + } + + meta, err := utils.MetaAccessor(out) + if err != nil { + return err + } + + // ensure the repo is set due to file provisioning, otherwise skip it + fileRepo, found := getProvisionedFileNameFromMeta(meta) + if !found { + return nil + } + + provisioning := &dashboardProvisioningWithUID{ + DashboardUID: hit.Name, + } + provisioning.Name = fileRepo + provisioning.ExternalID = meta.GetRepositoryPath() + provisioning.CheckSum = meta.GetRepositoryHash() + provisioning.DashboardID = meta.GetDeprecatedInternalID() // nolint:staticcheck + + updated, err := meta.GetRepositoryTimestamp() + if err != nil { + return err + } + if updated != nil { + provisioning.Updated = updated.Unix() + } + + mu.Lock() + dashs = append(dashs, provisioning) + mu.Unlock() + + return nil + }) + }(h) + } + + if err := g.Wait(); err != nil { + return nil, err + } + + return dashs, nil +} + func (dr *DashboardServiceImpl) searchDashboardsThroughK8s(ctx context.Context, query *dashboards.FindPersistedDashboardsQuery) ([]*dashboards.Dashboard, error) { response, err := dr.searchDashboardsThroughK8sRaw(ctx, query) if err != nil { @@ -1681,6 +2040,8 @@ func (dr *DashboardServiceImpl) UnstructuredToLegacyDashboard(ctx context.Contex out.Deleted = obj.GetDeletionTimestamp().Time } + out.PluginID = GetPluginIDFromMeta(obj) + creator, err := dr.getUserFromMeta(ctx, obj.GetCreatedBy()) if err != nil { return nil, err @@ -1705,10 +2066,6 @@ func (dr *DashboardServiceImpl) UnstructuredToLegacyDashboard(ctx context.Contex out.GnetID = gnetID } - if pluginID, ok := spec["plugin_id"].(string); ok { - out.PluginID = pluginID - } - if isFolder, ok := spec["is_folder"].(bool); ok { out.IsFolder = isFolder } @@ -1747,6 +2104,42 @@ func (dr *DashboardServiceImpl) getUser(ctx context.Context, uid string) (*user. return dr.userService.GetByUID(ctx, &user.GetUserByUIDQuery{UID: uid}) } +var pluginIDRepoName = "plugin" +var fileProvisionedRepoPrefix = "file:" + +func setPluginID(obj unstructured.Unstructured, pluginID string) { + if pluginID == "" { + return + } + + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[utils.AnnoKeyRepoName] = pluginIDRepoName + annotations[utils.AnnoKeyRepoPath] = pluginID + obj.SetAnnotations(annotations) +} + +func provisionedFileNameWithPrefix(name string) string { + if name == "" { + return "" + } + + return fileProvisionedRepoPrefix + name +} + +func getProvisionedFileNameFromMeta(obj utils.GrafanaMetaAccessor) (string, bool) { + return strings.CutPrefix(obj.GetRepositoryName(), fileProvisionedRepoPrefix) +} + +func GetPluginIDFromMeta(obj utils.GrafanaMetaAccessor) string { + if obj.GetRepositoryName() == pluginIDRepoName { + return obj.GetRepositoryPath() + } + return "" +} + func LegacySaveCommandToUnstructured(cmd *dashboards.SaveDashboardCommand, namespace string) (unstructured.Unstructured, error) { uid := cmd.GetDashboardModel().UID if uid == "" { diff --git a/pkg/services/dashboards/service/dashboard_service_test.go b/pkg/services/dashboards/service/dashboard_service_test.go index 959c7675ffe..4404dcfe8d0 100644 --- a/pkg/services/dashboards/service/dashboard_service_test.go +++ b/pkg/services/dashboards/service/dashboard_service_test.go @@ -2,6 +2,7 @@ package service import ( "context" + "fmt" "reflect" "testing" "time" @@ -135,7 +136,8 @@ func TestDashboardService(t *testing.T) { dto := &dashboards.SaveDashboardDTO{} t.Run("Should not return validation error if dashboard is provisioned", func(t *testing.T) { - fakeStore.On("SaveProvisionedDashboard", mock.Anything, mock.AnythingOfType("dashboards.SaveDashboardCommand"), mock.AnythingOfType("*dashboards.DashboardProvisioning")).Return(&dashboards.Dashboard{Data: simplejson.New()}, nil).Once() + fakeStore.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.AnythingOfType("*dashboards.DashboardProvisioning")).Return(nil).Once() + fakeStore.On("SaveDashboard", mock.Anything, mock.AnythingOfType("dashboards.SaveDashboardCommand")).Return(&dashboards.Dashboard{Data: simplejson.New()}, nil).Once() dto.Dashboard = dashboards.NewDashboard("Dash") dto.Dashboard.SetID(3) @@ -145,7 +147,8 @@ func TestDashboardService(t *testing.T) { }) t.Run("Should override invalid refresh interval if dashboard is provisioned", func(t *testing.T) { - fakeStore.On("SaveProvisionedDashboard", mock.Anything, mock.AnythingOfType("dashboards.SaveDashboardCommand"), mock.AnythingOfType("*dashboards.DashboardProvisioning")).Return(&dashboards.Dashboard{Data: simplejson.New()}, nil).Once() + fakeStore.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.AnythingOfType("*dashboards.DashboardProvisioning")).Return(nil).Once() + fakeStore.On("SaveDashboard", mock.Anything, mock.AnythingOfType("dashboards.SaveDashboardCommand")).Return(&dashboards.Dashboard{Data: simplejson.New()}, nil).Once() oldRefreshInterval := service.cfg.MinRefreshInterval service.cfg.MinRefreshInterval = "5m" @@ -246,7 +249,10 @@ func (m *mockDashK8sCli) getClient(ctx context.Context, orgID int64) (dynamic.Re } func (m *mockDashK8sCli) getNamespace(orgID int64) string { - return "default" + if orgID == 1 { + return "default" + } + return fmt.Sprintf("orgs-%d", orgID) } func (m *mockDashK8sCli) getSearcher() resource.ResourceIndexClient { @@ -567,6 +573,617 @@ func TestGetAllDashboardsByOrgId(t *testing.T) { }) } +func TestGetProvisionedDashboardData(t *testing.T) { + fakeStore := dashboards.FakeDashboardStore{} + defer fakeStore.AssertExpectations(t) + service := &DashboardServiceImpl{ + cfg: setting.NewCfg(), + dashboardStore: &fakeStore, + orgService: &orgtest.FakeOrgService{ + ExpectedOrgs: []*org.OrgDTO{{ID: 1}, {ID: 2}}, + }, + } + + t.Run("Should fallback to dashboard store if Kubernetes feature flags are not enabled", func(t *testing.T) { + service.features = featuremgmt.WithFeatures() + fakeStore.On("GetProvisionedDashboardData", mock.Anything, "test").Return([]*dashboards.DashboardProvisioning{}, nil).Once() + dashboard, err := service.GetProvisionedDashboardData(context.Background(), "test") + require.NoError(t, err) + require.NotNil(t, dashboard) + fakeStore.AssertExpectations(t) + }) + + t.Run("Should use Kubernetes client if feature flags are enabled and get from relevant org", func(t *testing.T) { + ctx, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service) + k8sClientMock.On("getClient", mock.Anything, mock.Anything).Return(k8sResourceMock, true) + k8sResourceMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{ + "name": "uid", + "labels": map[string]any{ + utils.LabelKeyDeprecatedInternalID: "1", // nolint:staticcheck + }, + "annotations": map[string]any{ + utils.AnnoKeyRepoName: fileProvisionedRepoPrefix + "test", + utils.AnnoKeyRepoHash: "hash", + utils.AnnoKeyRepoPath: "path/to/file", + utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z", + }, + }, + "spec": map[string]any{ + "test": "test", + "version": int64(1), + "title": "testing slugify", + }, + }}, nil).Once() + repo := "test" + k8sClientMock.searcher.On("Search", + mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool { + // ensure the prefix is added to the query + return req.Options.Key.Namespace == "default" && req.Options.Fields[0].Values[0] == provisionedFileNameWithPrefix(repo) + })).Return(&resource.ResourceSearchResponse{ + Results: &resource.ResourceTable{ + Columns: []*resource.ResourceTableColumnDefinition{}, + Rows: []*resource.ResourceTableRow{}, + }, + TotalHits: 0, + }, nil).Once() + k8sClientMock.searcher.On("Search", mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool { + // ensure the prefix is added to the query + return req.Options.Key.Namespace == "orgs-2" && req.Options.Fields[0].Values[0] == provisionedFileNameWithPrefix(repo) + })).Return(&resource.ResourceSearchResponse{ + Results: &resource.ResourceTable{ + Columns: []*resource.ResourceTableColumnDefinition{ + { + Name: "title", + }, + { + Name: "folder", + }, + }, + Rows: []*resource.ResourceTableRow{ + { + Key: &resource.ResourceKey{ + Name: "uid", + Resource: "dashboard", + }, + Cells: [][]byte{ + []byte("Dashboard 1"), + []byte("folder 1"), + }, + }, + }, + }, + TotalHits: 1, + }, nil).Once() + dashes, err := service.GetProvisionedDashboardData(ctx, repo) + require.NoError(t, err) + require.Len(t, dashes, 1) + require.Equal(t, dashes[0], &dashboards.DashboardProvisioning{ + ID: 0, + DashboardID: 1, + Name: "test", + ExternalID: "path/to/file", + CheckSum: "hash", + Updated: 1735689600, + }) + k8sClientMock.AssertExpectations(t) + }) +} + +func TestGetProvisionedDashboardDataByDashboardID(t *testing.T) { + fakeStore := dashboards.FakeDashboardStore{} + defer fakeStore.AssertExpectations(t) + service := &DashboardServiceImpl{ + cfg: setting.NewCfg(), + dashboardStore: &fakeStore, + orgService: &orgtest.FakeOrgService{ + ExpectedOrgs: []*org.OrgDTO{{ID: 1}, {ID: 2}}, + }, + } + + t.Run("Should fallback to dashboard store if Kubernetes feature flags are not enabled", func(t *testing.T) { + service.features = featuremgmt.WithFeatures() + fakeStore.On("GetProvisionedDataByDashboardID", mock.Anything, int64(1)).Return(&dashboards.DashboardProvisioning{}, nil).Once() + dashboard, err := service.GetProvisionedDashboardDataByDashboardID(context.Background(), 1) + require.NoError(t, err) + require.NotNil(t, dashboard) + fakeStore.AssertExpectations(t) + }) + + t.Run("Should use Kubernetes client if feature flags are enabled and get from whatever org it is in", func(t *testing.T) { + ctx, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service) + k8sClientMock.On("getClient", mock.Anything, mock.Anything).Return(k8sResourceMock, true) + k8sResourceMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{ + "name": "uid", + "labels": map[string]any{ + utils.LabelKeyDeprecatedInternalID: "1", // nolint:staticcheck + }, + "annotations": map[string]any{ + utils.AnnoKeyRepoName: fileProvisionedRepoPrefix + "test", + utils.AnnoKeyRepoHash: "hash", + utils.AnnoKeyRepoPath: "path/to/file", + utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z", + }, + }, + "spec": map[string]any{ + "test": "test", + "version": int64(1), + "title": "testing slugify", + }, + }}, nil) + k8sClientMock.searcher.On("Search", mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool { + return req.Options.Key.Namespace == "default" + })).Return(&resource.ResourceSearchResponse{ + Results: &resource.ResourceTable{ + Columns: []*resource.ResourceTableColumnDefinition{}, + Rows: []*resource.ResourceTableRow{}, + }, + TotalHits: 0, + }, nil) + k8sClientMock.searcher.On("Search", mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool { + return req.Options.Key.Namespace == "orgs-2" + })).Return(&resource.ResourceSearchResponse{ + Results: &resource.ResourceTable{ + Columns: []*resource.ResourceTableColumnDefinition{ + { + Name: "title", + }, + { + Name: "folder", + }, + }, + Rows: []*resource.ResourceTableRow{ + { + Key: &resource.ResourceKey{ + Name: "uid", + Resource: "dashboard", + }, + Cells: [][]byte{ + []byte("Dashboard 1"), + []byte("folder 1"), + }, + }, + }, + }, + TotalHits: 1, + }, nil) + dash, err := service.GetProvisionedDashboardDataByDashboardID(ctx, 1) + require.NoError(t, err) + require.Equal(t, dash, &dashboards.DashboardProvisioning{ + ID: 0, + DashboardID: 1, + Name: "test", + ExternalID: "path/to/file", + CheckSum: "hash", + Updated: 1735689600, + }) + k8sClientMock.AssertExpectations(t) + }) +} + +func TestGetProvisionedDashboardDataByDashboardUID(t *testing.T) { + fakeStore := dashboards.FakeDashboardStore{} + defer fakeStore.AssertExpectations(t) + service := &DashboardServiceImpl{ + cfg: setting.NewCfg(), + dashboardStore: &fakeStore, + orgService: &orgtest.FakeOrgService{ + ExpectedOrgs: []*org.OrgDTO{{ID: 1}, {ID: 2}}, + }, + } + + t.Run("Should fallback to dashboard store if Kubernetes feature flags are not enabled", func(t *testing.T) { + service.features = featuremgmt.WithFeatures() + fakeStore.On("GetProvisionedDataByDashboardUID", mock.Anything, int64(1), "test").Return(&dashboards.DashboardProvisioning{}, nil).Once() + dashboard, err := service.GetProvisionedDashboardDataByDashboardUID(context.Background(), 1, "test") + require.NoError(t, err) + require.NotNil(t, dashboard) + fakeStore.AssertExpectations(t) + }) + + t.Run("Should use Kubernetes client if feature flags are enabled", func(t *testing.T) { + ctx, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service) + k8sClientMock.On("getClient", mock.Anything, mock.Anything).Return(k8sResourceMock, true) + k8sResourceMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{ + "name": "uid", + "labels": map[string]any{ + utils.LabelKeyDeprecatedInternalID: "1", // nolint:staticcheck + }, + "annotations": map[string]any{ + utils.AnnoKeyRepoName: fileProvisionedRepoPrefix + "test", + utils.AnnoKeyRepoHash: "hash", + utils.AnnoKeyRepoPath: "path/to/file", + utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z", + }, + }, + "spec": map[string]any{ + "test": "test", + "version": int64(1), + "title": "testing slugify", + }, + }}, nil).Once() + k8sClientMock.searcher.On("Search", mock.Anything).Return(&resource.ResourceSearchResponse{ + Results: &resource.ResourceTable{ + Columns: []*resource.ResourceTableColumnDefinition{ + { + Name: "title", + }, + { + Name: "folder", + }, + }, + Rows: []*resource.ResourceTableRow{ + { + Key: &resource.ResourceKey{ + Name: "uid", + Resource: "dashboard", + }, + Cells: [][]byte{ + []byte("Dashboard 1"), + []byte("folder 1"), + }, + }, + }, + }, + TotalHits: 1, + }, nil).Once() + dash, err := service.GetProvisionedDashboardDataByDashboardUID(ctx, 1, "uid") + require.NoError(t, err) + require.Equal(t, dash, &dashboards.DashboardProvisioning{ + ID: 0, + DashboardID: 1, + Name: "test", + ExternalID: "path/to/file", + CheckSum: "hash", + Updated: 1735689600, + }) + k8sClientMock.AssertExpectations(t) + }) +} + +func TestDeleteOrphanedProvisionedDashboards(t *testing.T) { + fakeStore := dashboards.FakeDashboardStore{} + defer fakeStore.AssertExpectations(t) + service := &DashboardServiceImpl{ + cfg: setting.NewCfg(), + dashboardStore: &fakeStore, + orgService: &orgtest.FakeOrgService{ + ExpectedOrgs: []*org.OrgDTO{{ID: 1}, {ID: 2}}, + }, + } + + t.Run("Should fallback to dashboard store if Kubernetes feature flags are not enabled", func(t *testing.T) { + service.features = featuremgmt.WithFeatures() + fakeStore.On("DeleteOrphanedProvisionedDashboards", mock.Anything, &dashboards.DeleteOrphanedProvisionedDashboardsCommand{ + ReaderNames: []string{"test"}, + }).Return(nil).Once() + err := service.DeleteOrphanedProvisionedDashboards(context.Background(), &dashboards.DeleteOrphanedProvisionedDashboardsCommand{ + ReaderNames: []string{"test"}, + }) + require.NoError(t, err) + fakeStore.AssertExpectations(t) + }) + + t.Run("Should use Kubernetes client if feature flags are enabled, delete across all orgs, but only delete file based provisioned dashboards", func(t *testing.T) { + _, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service) + k8sClientMock.On("getClient", mock.Anything, mock.Anything).Return(k8sResourceMock, true) + k8sResourceMock.On("Delete", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + fakeStore.On("CleanupAfterDelete", mock.Anything, &dashboards.DeleteDashboardCommand{UID: "uid", OrgID: 1}).Return(nil).Once() + fakeStore.On("CleanupAfterDelete", mock.Anything, &dashboards.DeleteDashboardCommand{UID: "uid3", OrgID: 2}).Return(nil).Once() + k8sResourceMock.On("Get", mock.Anything, "uid", mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{ + "name": "uid", + "annotations": map[string]any{ + utils.AnnoKeyRepoName: fileProvisionedRepoPrefix + "orphaned", + utils.AnnoKeyRepoHash: "hash", + utils.AnnoKeyRepoPath: "path/to/file", + utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z", + }, + }, + "spec": map[string]any{}, + }}, nil).Once() + // should not delete this one, because it does not start with "file:" + k8sResourceMock.On("Get", mock.Anything, "uid2", mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{ + "name": "uid2", + "annotations": map[string]any{ + utils.AnnoKeyRepoName: "plugin", + utils.AnnoKeyRepoHash: "app", + }, + }, + "spec": map[string]any{}, + }}, nil).Once() + + k8sResourceMock.On("Get", mock.Anything, "uid3", mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{ + "name": "uid3", + "annotations": map[string]any{ + utils.AnnoKeyRepoName: fileProvisionedRepoPrefix + "orphaned", + utils.AnnoKeyRepoHash: "hash", + utils.AnnoKeyRepoPath: "path/to/file", + utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z", + }, + }, + "spec": map[string]any{}, + }}, nil).Once() + k8sClientMock.searcher.On("Search", mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool { + return req.Options.Key.Namespace == "default" && req.Options.Fields[0].Key == "repo.name" && req.Options.Fields[0].Values[0] == provisionedFileNameWithPrefix("test") && + req.Options.Fields[0].Operator == "notin" + })).Return(&resource.ResourceSearchResponse{ + Results: &resource.ResourceTable{ + Columns: []*resource.ResourceTableColumnDefinition{ + { + Name: "title", + }, + { + Name: "folder", + }, + }, + Rows: []*resource.ResourceTableRow{ + { + Key: &resource.ResourceKey{ + Name: "uid", + Resource: "dashboard", + }, + Cells: [][]byte{ + []byte("Dashboard 1"), + []byte("folder 1"), + }, + }, + }, + }, + TotalHits: 1, + }, nil).Once() + + k8sClientMock.searcher.On("Search", mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool { + return req.Options.Key.Namespace == "orgs-2" && req.Options.Fields[0].Key == "repo.name" && req.Options.Fields[0].Values[0] == provisionedFileNameWithPrefix("test") && + req.Options.Fields[0].Operator == "notin" + })).Return(&resource.ResourceSearchResponse{ + Results: &resource.ResourceTable{ + Columns: []*resource.ResourceTableColumnDefinition{ + { + Name: "title", + }, + { + Name: "folder", + }, + }, + Rows: []*resource.ResourceTableRow{ + { + Key: &resource.ResourceKey{ + Name: "uid2", + Resource: "dashboard", + }, + Cells: [][]byte{ + []byte("Dashboard 2"), + []byte("folder 2"), + }, + }, + { + Key: &resource.ResourceKey{ + Name: "uid3", + Resource: "dashboard", + }, + Cells: [][]byte{ + []byte("Dashboard 3"), + []byte("folder 3"), + }, + }, + }, + }, + TotalHits: 2, + }, nil).Once() + err := service.DeleteOrphanedProvisionedDashboards(context.Background(), &dashboards.DeleteOrphanedProvisionedDashboardsCommand{ + ReaderNames: []string{"test"}, + }) + require.NoError(t, err) + k8sClientMock.AssertExpectations(t) + }) +} + +func TestUnprovisionDashboard(t *testing.T) { + fakeStore := dashboards.FakeDashboardStore{} + defer fakeStore.AssertExpectations(t) + service := &DashboardServiceImpl{ + cfg: setting.NewCfg(), + dashboardStore: &fakeStore, + orgService: &orgtest.FakeOrgService{ + ExpectedOrgs: []*org.OrgDTO{{ID: 1}, {ID: 2}}, + }, + } + + t.Run("Should fallback to dashboard store if Kubernetes feature flags are not enabled", func(t *testing.T) { + service.features = featuremgmt.WithFeatures() + fakeStore.On("UnprovisionDashboard", mock.Anything, int64(1)).Return(nil).Once() + err := service.UnprovisionDashboard(context.Background(), 1) + require.NoError(t, err) + fakeStore.AssertExpectations(t) + }) + + t.Run("Should use Kubernetes client if feature flags are enabled - should remove annotations", func(t *testing.T) { + ctx, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service) + k8sClientMock.On("getClient", mock.Anything, mock.Anything).Return(k8sResourceMock, true) + dash := &unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{ + "name": "uid", + "annotations": map[string]any{ + utils.AnnoKeyRepoName: fileProvisionedRepoPrefix + "test", + utils.AnnoKeyRepoHash: "hash", + utils.AnnoKeyRepoPath: "path/to/file", + utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z", + }, + }, + "spec": map[string]any{}, + }} + k8sResourceMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(dash, nil) + dashWithoutAnnotations := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "dashboard.grafana.app/v0alpha1", + "kind": "Dashboard", + "metadata": map[string]any{ + "name": "uid", + "namespace": "default", + "annotations": map[string]any{}, + }, + "spec": map[string]any{ + "uid": "uid", + "version": 1, + }, + }} + // should update it to be without annotations + k8sResourceMock.On("Update", mock.Anything, dashWithoutAnnotations, mock.Anything, mock.Anything).Return(dashWithoutAnnotations, nil) + k8sClientMock.searcher.On("Search", mock.Anything).Return(&resource.ResourceSearchResponse{ + Results: &resource.ResourceTable{ + Columns: []*resource.ResourceTableColumnDefinition{ + { + Name: "title", + }, + { + Name: "folder", + }, + }, + Rows: []*resource.ResourceTableRow{ + { + Key: &resource.ResourceKey{ + Name: "uid", + Resource: "dashboard", + }, + Cells: [][]byte{ + []byte("Dashboard 1"), + []byte("folder 1"), + }, + }, + }, + }, + TotalHits: 1, + }, nil) + err := service.UnprovisionDashboard(ctx, 1) + require.NoError(t, err) + k8sClientMock.AssertExpectations(t) + }) +} + +func TestGetDashboardsByPluginID(t *testing.T) { + fakeStore := dashboards.FakeDashboardStore{} + defer fakeStore.AssertExpectations(t) + service := &DashboardServiceImpl{ + cfg: setting.NewCfg(), + dashboardStore: &fakeStore, + } + + query := &dashboards.GetDashboardsByPluginIDQuery{ + PluginID: "testing", + OrgID: 1, + } + t.Run("Should fallback to dashboard store if Kubernetes feature flags are not enabled", func(t *testing.T) { + service.features = featuremgmt.WithFeatures() + fakeStore.On("GetDashboardsByPluginID", mock.Anything, mock.Anything).Return([]*dashboards.Dashboard{}, nil).Once() + _, err := service.GetDashboardsByPluginID(context.Background(), query) + require.NoError(t, err) + fakeStore.AssertExpectations(t) + }) + + t.Run("Should use Kubernetes client if feature flags are enabled", func(t *testing.T) { + ctx, k8sClientMock, _ := setupK8sDashboardTests(service) + k8sClientMock.searcher.On("Search", mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool { + return req.Options.Fields[0].Key == "repo.name" && req.Options.Fields[0].Values[0] == "plugin" && + req.Options.Fields[1].Key == "repo.path" && req.Options.Fields[1].Values[0] == "testing" + })).Return(&resource.ResourceSearchResponse{ + Results: &resource.ResourceTable{ + Columns: []*resource.ResourceTableColumnDefinition{ + { + Name: "title", + }, + { + Name: "folder", + }, + }, + Rows: []*resource.ResourceTableRow{ + { + Key: &resource.ResourceKey{ + Name: "uid", + Resource: "dashboard", + }, + Cells: [][]byte{ + []byte("Dashboard 1"), + []byte("folder 1"), + }, + }, + }, + }, + TotalHits: 1, + }, nil) + dashes, err := service.GetDashboardsByPluginID(ctx, query) + require.NoError(t, err) + require.Len(t, dashes, 1) + k8sClientMock.AssertExpectations(t) + }) +} + +func TestSaveProvisionedDashboard(t *testing.T) { + fakeStore := dashboards.FakeDashboardStore{} + defer fakeStore.AssertExpectations(t) + service := &DashboardServiceImpl{ + cfg: setting.NewCfg(), + dashboardStore: &fakeStore, + log: log.NewNopLogger(), + } + + origNewDashboardGuardian := guardian.New + defer func() { guardian.New = origNewDashboardGuardian }() + guardian.MockDashboardGuardian(&guardian.FakeDashboardGuardian{CanSaveValue: true}) + + query := &dashboards.SaveDashboardDTO{ + OrgID: 1, + User: &user.SignedInUser{UserID: 1}, + Dashboard: &dashboards.Dashboard{ + UID: "uid", + Title: "testing slugify", + Slug: "testing-slugify", + OrgID: 1, + Data: simplejson.NewFromAny(map[string]any{"test": "test", "title": "testing slugify", "uid": "uid"}), + }, + } + + t.Run("Should fallback to dashboard store if Kubernetes feature flags are not enabled", func(t *testing.T) { + service.features = featuremgmt.WithFeatures() + fakeStore.On("GetDashboard", mock.Anything, mock.Anything).Return(&dashboards.Dashboard{}, nil) + fakeStore.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.Anything).Return(nil) + fakeStore.On("SaveDashboard", mock.Anything, mock.Anything, mock.Anything).Return(&dashboards.Dashboard{}, nil) + dashboard, err := service.SaveProvisionedDashboard(context.Background(), query, &dashboards.DashboardProvisioning{}) + require.NoError(t, err) + require.NotNil(t, dashboard) + fakeStore.AssertExpectations(t) + }) + + dashboardUnstructured := unstructured.Unstructured{Object: map[string]any{ + "metadata": map[string]any{ + "name": "uid", + }, + "spec": map[string]any{ + "test": "test", + "version": int64(1), + "title": "testing slugify", + }, + }} + + t.Run("Should use Kubernetes create if feature flags are enabled", func(t *testing.T) { + ctx, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service) + fakeStore.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.Anything).Return(nil) + k8sClientMock.On("getClient", mock.Anything, int64(1)).Return(k8sResourceMock, true) + k8sResourceMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) + k8sResourceMock.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&dashboardUnstructured, nil) + + dashboard, err := service.SaveProvisionedDashboard(ctx, query, &dashboards.DashboardProvisioning{}) + require.NoError(t, err) + require.NotNil(t, dashboard) + k8sClientMock.AssertExpectations(t) + // ensure the provisioning data is still saved to the db + fakeStore.AssertExpectations(t) + }) +} + func TestSaveDashboard(t *testing.T) { fakeStore := dashboards.FakeDashboardStore{} defer fakeStore.AssertExpectations(t) @@ -667,22 +1284,19 @@ func TestDeleteDashboard(t *testing.T) { t.Run("Should use Kubernetes client if feature flags are enabled", func(t *testing.T) { ctx, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service) k8sClientMock.On("getClient", mock.Anything, int64(1)).Return(k8sResourceMock, true).Once() - k8sResourceMock.On("Delete", mock.Anything, mock.Anything, metav1.DeleteOptions{ - GracePeriodSeconds: nil, - }, mock.Anything).Return(nil).Once() + k8sResourceMock.On("Delete", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + fakeStore.On("CleanupAfterDelete", mock.Anything, mock.Anything).Return(nil).Once() err := service.DeleteDashboard(ctx, 1, "uid", 1) require.NoError(t, err) k8sClientMock.AssertExpectations(t) }) - t.Run("When UID is not passed in for provisioned dashboards, should retrieve that first and set grace period to zero", func(t *testing.T) { + t.Run("If UID is not passed in, it should retrieve that first", func(t *testing.T) { ctx, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service) - zeroInt64 := int64(0) k8sClientMock.On("getClient", mock.Anything, int64(1)).Return(k8sResourceMock, true).Once() - k8sResourceMock.On("Delete", mock.Anything, mock.Anything, metav1.DeleteOptions{ - GracePeriodSeconds: &zeroInt64, - }, mock.Anything).Return(nil).Once() + k8sResourceMock.On("Delete", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + fakeStore.On("CleanupAfterDelete", mock.Anything, mock.Anything).Return(nil).Once() k8sClientMock.searcher.On("Search", mock.Anything).Return(&resource.ResourceSearchResponse{ Results: &resource.ResourceTable{ Columns: []*resource.ResourceTableColumnDefinition{ @@ -708,7 +1322,7 @@ func TestDeleteDashboard(t *testing.T) { }, TotalHits: 1, }, nil) - err := service.DeleteProvisionedDashboard(ctx, 1, 1) + err := service.DeleteDashboard(ctx, 1, "", 1) require.NoError(t, err) k8sClientMock.AssertExpectations(t) k8sClientMock.searcher.AssertExpectations(t) diff --git a/pkg/services/dashboards/store_mock.go b/pkg/services/dashboards/store_mock.go index 0c8e5b190f7..7513af3125a 100644 --- a/pkg/services/dashboards/store_mock.go +++ b/pkg/services/dashboards/store_mock.go @@ -94,6 +94,24 @@ func (_m *FakeDashboardStore) DeleteDashboard(ctx context.Context, cmd *DeleteDa return r0 } +// CleanupAfterDelete provides a mock function with given fields: ctx, cmd +func (_m *FakeDashboardStore) CleanupAfterDelete(ctx context.Context, cmd *DeleteDashboardCommand) error { + ret := _m.Called(ctx, cmd) + + if len(ret) == 0 { + panic("no return value specified for CleanupAfterDelete") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *DeleteDashboardCommand) error); ok { + r0 = rf(ctx, cmd) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // DeleteAllDashboards provides a mock function with given fields: ctx, orgID func (_m *FakeDashboardStore) DeleteAllDashboards(ctx context.Context, orgID int64) error { ret := _m.Called(ctx, orgID) @@ -208,7 +226,6 @@ func (_m *FakeDashboardStore) GetAllDashboards(ctx context.Context) ([]*Dashboar return r0, r1 } - // GetAllDashboardsByOrgId provides a mock function with given fields: ctx func (_m *FakeDashboardStore) GetAllDashboardsByOrgId(ctx context.Context, orgID int64) ([]*Dashboard, error) { ret := _m.Called(ctx, orgID) @@ -588,34 +605,22 @@ func (_m *FakeDashboardStore) SaveDashboard(ctx context.Context, cmd SaveDashboa return r0, r1 } -// SaveProvisionedDashboard provides a mock function with given fields: ctx, cmd, provisioning -func (_m *FakeDashboardStore) SaveProvisionedDashboard(ctx context.Context, cmd SaveDashboardCommand, provisioning *DashboardProvisioning) (*Dashboard, error) { - ret := _m.Called(ctx, cmd, provisioning) +// SaveProvisionedDashboard provides a mock function with given fields: ctx, dash, provisioning +func (_m *FakeDashboardStore) SaveProvisionedDashboard(ctx context.Context, dash *Dashboard, provisioning *DashboardProvisioning) error { + ret := _m.Called(ctx, dash, provisioning) if len(ret) == 0 { panic("no return value specified for SaveProvisionedDashboard") } - var r0 *Dashboard - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, SaveDashboardCommand, *DashboardProvisioning) (*Dashboard, error)); ok { - return rf(ctx, cmd, provisioning) - } - if rf, ok := ret.Get(0).(func(context.Context, SaveDashboardCommand, *DashboardProvisioning) *Dashboard); ok { - r0 = rf(ctx, cmd, provisioning) + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *Dashboard, *DashboardProvisioning) error); ok { + r0 = rf(ctx, dash, provisioning) } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*Dashboard) - } + r0 = ret.Error(0) } - if rf, ok := ret.Get(1).(func(context.Context, SaveDashboardCommand, *DashboardProvisioning) error); ok { - r1 = rf(ctx, cmd, provisioning) - } else { - r1 = ret.Error(1) - } - - return r0, r1 + return r0 } // SoftDeleteDashboard provides a mock function with given fields: ctx, orgID, dashboardUid diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go index 7baa748c05a..1545a2fcfb2 100644 --- a/pkg/storage/unified/search/bleve.go +++ b/pkg/storage/unified/search/bleve.go @@ -625,6 +625,19 @@ func requirementQuery(req *resource.Requirement, prefix string) (query.Query, *r return query.NewDisjunctionQuery(disjuncts), nil case selection.NotIn: + boolQuery := bleve.NewBooleanQuery() + + var mustNotQueries []query.Query + for _, value := range req.Values { + mustNotQueries = append(mustNotQueries, bleve.NewMatchQuery(value)) + } + boolQuery.AddMustNot(mustNotQueries...) + + // must still have a value + notEmptyQuery := bleve.NewWildcardQuery("*") + boolQuery.AddMust(notEmptyQuery) + + return boolQuery, nil } return nil, resource.NewBadRequestError( fmt.Sprintf("unsupported query operation (%s %s %v)", req.Key, req.Operator, req.Values),