From a5c14db051dadef493c32b7f1afe37cb2966c7ea Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 27 Jan 2025 19:37:28 +0300 Subject: [PATCH] Folders: Convert between unstructured and legacy (#99504) --- pkg/api/dtos/folder.go | 8 + pkg/api/folder.go | 10 +- pkg/registry/apis/folders/conversions.go | 141 ++++++------------ pkg/registry/apis/folders/conversions_test.go | 62 ++++++++ .../folder/folderimpl/unifiedstore.go | 53 ++++--- pkg/services/folder/model.go | 5 + public/api-enterprise-spec.json | 42 +++--- public/api-merged.json | 8 + public/openapi3.json | 8 + 9 files changed, 194 insertions(+), 143 deletions(-) create mode 100644 pkg/registry/apis/folders/conversions_test.go diff --git a/pkg/api/dtos/folder.go b/pkg/api/dtos/folder.go index 5830187ce78..19df61f4f68 100644 --- a/pkg/api/dtos/folder.go +++ b/pkg/api/dtos/folder.go @@ -28,6 +28,10 @@ type Folder struct { ParentUID string `json:"parentUid,omitempty"` // the parent folders starting from the root going down Parents []Folder `json:"parents,omitempty"` + + // When the folder belongs to a repository + // NOTE: this is only populated when folders are managed by unified storage + Repository string `json:"repository,omitempty"` } type FolderSearchHit struct { @@ -35,4 +39,8 @@ type FolderSearchHit struct { UID string `json:"uid" xorm:"uid"` Title string `json:"title"` ParentUID string `json:"parentUid,omitempty"` + + // When the folder belongs to a repository + // NOTE: this is only populated when folders are managed by unified storage + Repository string `json:"repository,omitempty"` } diff --git a/pkg/api/folder.go b/pkg/api/folder.go index af38eb9de2c..eb4cb853dda 100644 --- a/pkg/api/folder.go +++ b/pkg/api/folder.go @@ -94,10 +94,11 @@ func (hs *HTTPServer) GetFolders(c *contextmodel.ReqContext) response.Response { hits := make([]dtos.FolderSearchHit, 0) for _, f := range folders { hits = append(hits, dtos.FolderSearchHit{ - ID: f.ID, // nolint:staticcheck - UID: f.UID, - Title: f.Title, - ParentUID: f.ParentUID, + ID: f.ID, // nolint:staticcheck + UID: f.UID, + Title: f.Title, + ParentUID: f.ParentUID, + Repository: f.Repository, }) metrics.MFolderIDsAPICount.WithLabelValues(metrics.GetFolders).Inc() } @@ -425,6 +426,7 @@ func (hs *HTTPServer) newToFolderDto(c *contextmodel.ReqContext, f *folder.Folde Version: f.Version, AccessControl: acMetadata, ParentUID: f.ParentUID, + Repository: f.Repository, }, nil } diff --git a/pkg/registry/apis/folders/conversions.go b/pkg/registry/apis/folders/conversions.go index 470b22ff0f6..d5e2f50901d 100644 --- a/pkg/registry/apis/folders/conversions.go +++ b/pkg/registry/apis/folders/conversions.go @@ -2,11 +2,11 @@ package folders import ( "fmt" - "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + authlib "github.com/grafana/authlib/types" "github.com/grafana/grafana/pkg/apimachinery/utils" "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" "github.com/grafana/grafana/pkg/infra/slugify" @@ -26,98 +26,65 @@ func LegacyCreateCommandToUnstructured(cmd *folder.CreateFolderCommand) (*unstru }, }, } - // #TODO: let's see if we need to set the json field to "-" + + meta, err := utils.MetaAccessor(obj) + if err != nil { + return nil, err + } + if cmd.UID == "" { cmd.UID = util.GenerateShortUID() } - obj.SetName(cmd.UID) - - if err := setParentUID(obj, cmd.ParentUID); err != nil { - return &unstructured.Unstructured{}, err - } + meta.SetName(cmd.UID) + meta.SetFolder(cmd.ParentUID) return obj, nil } -func LegacyUpdateCommandToUnstructured(obj *unstructured.Unstructured, cmd *folder.UpdateFolderCommand) (*unstructured.Unstructured, error) { - spec, ok := obj.Object["spec"].(map[string]any) - if !ok { - return &unstructured.Unstructured{}, fmt.Errorf("could not convert object to folder") - } - if cmd.NewTitle != nil { - spec["title"] = cmd.NewTitle - } - if cmd.NewDescription != nil { - spec["description"] = cmd.NewDescription - } - if cmd.NewParentUID != nil { - if err := setParentUID(obj, *cmd.NewParentUID); err != nil { - return &unstructured.Unstructured{}, err - } - } - - return obj, nil -} - -func LegacyMoveCommandToUnstructured(obj *unstructured.Unstructured, cmd folder.MoveFolderCommand) (*unstructured.Unstructured, error) { - if err := setParentUID(obj, cmd.NewParentUID); err != nil { - return &unstructured.Unstructured{}, err - } - - return obj, nil -} - -func UnstructuredToLegacyFolder(item unstructured.Unstructured, orgID int64) (*folder.Folder, string) { - // #TODO reduce duplication of the different conversion functions - spec := item.Object["spec"].(map[string]any) - uid := item.GetName() - title := spec["title"].(string) - - meta, err := utils.MetaAccessor(&item) +func UnstructuredToLegacyFolder(item *unstructured.Unstructured) (*folder.Folder, error) { + meta, err := utils.MetaAccessor(item) if err != nil { - return nil, "" + return nil, err } - id := meta.GetDeprecatedInternalID() // nolint:staticcheck - - created, err := getCreated(meta) - if err != nil { - return nil, "" + info, _ := authlib.ParseNamespace(meta.GetNamespace()) + if info.OrgID < 0 { + info.OrgID = 1 // This resolves all test cases that assume org 1 } - // avoid panic - var createdTime time.Time - if created != nil { - // #TODO Fix this time format. The legacy time format seems to be along the lines of time.Now() - // which includes a part that represents a fraction of a second. Format should be "2024-09-12T15:37:41.09466+02:00" - createdTime = (*created).UTC() + title, _, _ := unstructured.NestedString(item.Object, "spec", "title") + description, _, _ := unstructured.NestedString(item.Object, "spec", "description") + + uid := meta.GetName() + url := "" + if uid != folder.RootFolder.UID { + slug := slugify.Slugify(title) + url = dashboards.GetFolderURL(uid, slug) } - url := getURL(meta, title) - - // RootFolder does not have URL - if uid == folder.RootFolder.UID { - url = "" + created := meta.GetCreationTimestamp().Time.UTC() + updated, _ := meta.GetUpdatedTimestamp() + if updated == nil { + updated = &created + } else { + tmp := updated.UTC() + updated = &tmp } - f := &folder.Folder{ - UID: uid, - Title: title, - ID: id, - ParentUID: meta.GetFolder(), - // #TODO add created by field if necessary - URL: url, - // #TODO get Created in format "2024-09-12T15:37:41.09466+02:00" - Created: createdTime, - // #TODO figure out whether we want to set "updated" and "updated by". Could replace with - // meta.GetUpdatedTimestamp() but it currently gets overwritten in prepareObjectForStorage(). - Updated: createdTime, - OrgID: orgID, - } - // CreatedBy needs to be returned separately because it's the user UID (string) but - // folder.Folder expects user ID (int64). - return f, meta.GetCreatedBy() - // #TODO figure out about adding version, parents, orgID fields + return &folder.Folder{ + UID: uid, + Title: title, + Description: description, + ID: meta.GetDeprecatedInternalID(), // nolint:staticcheck + ParentUID: meta.GetFolder(), + Version: int(meta.GetGeneration()), + Repository: meta.GetRepositoryName(), + + URL: url, + Created: created, + Updated: *updated, + OrgID: info.OrgID, + }, nil } func LegacyFolderToUnstructured(v *folder.Folder, namespacer request.NamespaceMapper) (*v0alpha1.Folder, error) { @@ -164,23 +131,3 @@ func convertToK8sResource(v *folder.Folder, namespacer request.NamespaceMapper) f.UID = gapiutil.CalculateClusterWideUID(f) return f, nil } - -func setParentUID(u *unstructured.Unstructured, parentUid string) error { - meta, err := utils.MetaAccessor(u) - if err != nil { - return err - } - meta.SetFolder(parentUid) - return nil -} - -func getURL(meta utils.GrafanaMetaAccessor, title string) string { - slug := slugify.Slugify(title) - uid := meta.GetName() - return dashboards.GetFolderURL(uid, slug) -} - -func getCreated(meta utils.GrafanaMetaAccessor) (*time.Time, error) { - created := meta.GetCreationTimestamp().Time - return &created, nil -} diff --git a/pkg/registry/apis/folders/conversions_test.go b/pkg/registry/apis/folders/conversions_test.go new file mode 100644 index 00000000000..f7bef8ba015 --- /dev/null +++ b/pkg/registry/apis/folders/conversions_test.go @@ -0,0 +1,62 @@ +package folders + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/grafana/grafana/pkg/services/folder" +) + +func TestFolderConversions(t *testing.T) { + input := &unstructured.Unstructured{} + err := input.UnmarshalJSON([]byte(`{ + "kind": "Folder", + "apiVersion": "folder.grafana.app/v0alpha1", + "metadata": { + "name": "be79sztagf20wd", + "namespace": "default", + "uid": "wfi3RARqQREzEKtUJCWurWevwbQ7i9ii0cA7JUIbMtEX", + "resourceVersion": "1734509107000", + "creationTimestamp": "2022-12-02T02:02:02Z", + "generation": 4, + "labels": { + "grafana.app/deprecatedInternalID": "234" + }, + "annotations": { + "grafana.app/folder": "parent-folder-name", + "grafana.app/updatedTimestamp": "2022-12-02T07:02:02Z", + "grafana.app/repoName": "example-repo", + "grafana.app/createdBy": "user:abc", + "grafana.app/updatedBy": "service:xyz" + } + }, + "spec": { + "title": "test folder", + "description": "Something set in the file" + } + }`)) + require.NoError(t, err) + + created, err := time.Parse(time.RFC3339, "2022-12-02T02:02:02Z") + created = created.UTC() + require.NoError(t, err) + + converted, err := UnstructuredToLegacyFolder(input) + require.NoError(t, err) + require.Equal(t, folder.Folder{ + ID: 234, + OrgID: 1, + Version: 4, + UID: "be79sztagf20wd", + ParentUID: "parent-folder-name", + Title: "test folder", + Description: "Something set in the file", + URL: "/dashboards/f/be79sztagf20wd/test-folder", + Repository: "example-repo", + Created: created, + Updated: created.Add(time.Hour * 5), + }, *converted) +} diff --git a/pkg/services/folder/folderimpl/unifiedstore.go b/pkg/services/folder/folderimpl/unifiedstore.go index 05f9819662b..fe835c16f7a 100644 --- a/pkg/services/folder/folderimpl/unifiedstore.go +++ b/pkg/services/folder/folderimpl/unifiedstore.go @@ -6,13 +6,15 @@ import ( "strings" "time" - "github.com/grafana/grafana/pkg/apimachinery/identity" apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" k8sUser "k8s.io/apiserver/pkg/authentication/user" k8sRequest "k8s.io/apiserver/pkg/endpoints/request" + "github.com/grafana/grafana/pkg/apimachinery/identity" + "github.com/grafana/grafana/pkg/apimachinery/utils" + "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" "github.com/grafana/grafana/pkg/infra/log" internalfolders "github.com/grafana/grafana/pkg/registry/apis/folders" @@ -58,7 +60,7 @@ func (ss *FolderUnifiedStoreImpl) Create(ctx context.Context, cmd folder.CreateF return nil, err } - folder, _ := internalfolders.UnstructuredToLegacyFolder(*out, cmd.SignedInUser.GetOrgID()) + folder, err := internalfolders.UnstructuredToLegacyFolder(out) if err != nil { return nil, err } @@ -106,10 +108,26 @@ func (ss *FolderUnifiedStoreImpl) Update(ctx context.Context, cmd folder.UpdateF if err != nil { return nil, err } + updated := obj.DeepCopy() - updated, err := internalfolders.LegacyUpdateCommandToUnstructured(obj, &cmd) - if err != nil { - return nil, err + if cmd.NewTitle != nil { + err = unstructured.SetNestedField(updated.Object, *cmd.NewTitle, "spec", "title") + if err != nil { + return nil, err + } + } + if cmd.NewDescription != nil { + err = unstructured.SetNestedField(updated.Object, *cmd.NewDescription, "spec", "description") + if err != nil { + return nil, err + } + } + if cmd.NewParentUID != nil { + meta, err := utils.MetaAccessor(updated) + if err != nil { + return nil, err + } + meta.SetFolder(*cmd.NewParentUID) } out, err := client.Update(ctx, updated, v1.UpdateOptions{}) @@ -117,12 +135,7 @@ func (ss *FolderUnifiedStoreImpl) Update(ctx context.Context, cmd folder.UpdateF return nil, err } - folder, _ := internalfolders.UnstructuredToLegacyFolder(*out, cmd.SignedInUser.GetOrgID()) - if err != nil { - return nil, err - } - - return folder, err + return internalfolders.UnstructuredToLegacyFolder(out) } // If WithFullpath is true it computes also the full path of a folder. @@ -164,8 +177,8 @@ func (ss *FolderUnifiedStoreImpl) Get(ctx context.Context, q folder.GetFolderQue } else if err != nil || out == nil { return nil, dashboards.ErrFolderNotFound } - dashFolder, _ := internalfolders.UnstructuredToLegacyFolder(*out, q.SignedInUser.GetOrgID()) - return dashFolder, nil + + return internalfolders.UnstructuredToLegacyFolder(out) } func (ss *FolderUnifiedStoreImpl) GetParents(ctx context.Context, q folder.GetParentsQuery) ([]*folder.Folder, error) { @@ -193,7 +206,7 @@ func (ss *FolderUnifiedStoreImpl) GetParents(ctx context.Context, q folder.GetPa return nil, err } - folder, _ := internalfolders.UnstructuredToLegacyFolder(*out, q.OrgID) + folder, err := internalfolders.UnstructuredToLegacyFolder(out) if err != nil { return nil, err } @@ -232,9 +245,9 @@ func (ss *FolderUnifiedStoreImpl) GetChildren(ctx context.Context, q folder.GetC hits := make([]*folder.Folder, 0) for _, item := range out.Items { // convert item to legacy folder format - f, _ := internalfolders.UnstructuredToLegacyFolder(item, q.OrgID) + f, err := internalfolders.UnstructuredToLegacyFolder(&item) if f == nil { - return nil, fmt.Errorf("unable covert unstructured item to legacy folder") + return nil, fmt.Errorf("unable covert unstructured item to legacy folder %w", err) } // it we are at root level, skip subfolder @@ -332,9 +345,9 @@ func (ss *FolderUnifiedStoreImpl) GetFolders(ctx context.Context, q folder.GetFo m := map[string]*folder.Folder{} for _, item := range out.Items { // convert item to legacy folder format - f, _ := internalfolders.UnstructuredToLegacyFolder(item, q.SignedInUser.GetOrgID()) + f, err := internalfolders.UnstructuredToLegacyFolder(&item) if f == nil { - return nil, fmt.Errorf("unable covert unstructured item to legacy folder") + return nil, fmt.Errorf("unable covert unstructured item to legacy folder %w", err) } m[f.UID] = f @@ -392,9 +405,9 @@ func (ss *FolderUnifiedStoreImpl) GetDescendants(ctx context.Context, orgID int6 nodes := map[string]*folder.Folder{} for _, item := range out.Items { // convert item to legacy folder format - f, _ := internalfolders.UnstructuredToLegacyFolder(item, orgID) + f, err := internalfolders.UnstructuredToLegacyFolder(&item) if f == nil { - return nil, fmt.Errorf("unable covert unstructured item to legacy folder") + return nil, fmt.Errorf("unable covert unstructured item to legacy folder %w", err) } nodes[f.UID] = f diff --git a/pkg/services/folder/model.go b/pkg/services/folder/model.go index ccadb46f568..dae38765773 100644 --- a/pkg/services/folder/model.go +++ b/pkg/services/folder/model.go @@ -53,6 +53,11 @@ type Folder struct { HasACL bool Fullpath string `xorm:"fullpath"` FullpathUIDs string `xorm:"fullpath_uids"` + + // When the folder belongs to a repository + // NOTE: this is only populated when folders are managed by unified storage + // This is not ever used by xorm, but the translation functions flow through this type + Repository string `json:"repository,omitempty"` } var GeneralFolder = Folder{ID: 0, Title: "General"} diff --git a/public/api-enterprise-spec.json b/public/api-enterprise-spec.json index 7090f1ff89b..7667a82756c 100644 --- a/public/api-enterprise-spec.json +++ b/public/api-enterprise-spec.json @@ -2837,7 +2837,6 @@ }, "AnnotationPanelFilter": { "type": "object", - "title": "AnnotationPanelFilter defines model for AnnotationPanelFilter.", "properties": { "exclude": { "description": "Should the specified panels be included or excluded", @@ -2848,7 +2847,7 @@ "type": "array", "items": { "type": "integer", - "format": "int64" + "format": "uint8" } } } @@ -2871,7 +2870,7 @@ "builtIn": { "description": "Set to 1 for the standard annotation query all dashboards have by default.", "type": "number", - "format": "float" + "format": "double" }, "datasource": { "$ref": "#/definitions/DataSourceRef" @@ -3438,20 +3437,10 @@ }, "CookiePreferences": { "type": "object", - "title": "CookiePreferences defines model for CookiePreferences.", "properties": { - "analytics": { - "type": "object", - "additionalProperties": {} - }, - "functional": { - "type": "object", - "additionalProperties": {} - }, - "performance": { - "type": "object", - "additionalProperties": {} - } + "analytics": {}, + "functional": {}, + "performance": {} } }, "CookieType": { @@ -4760,6 +4749,10 @@ "$ref": "#/definitions/Folder" } }, + "repository": { + "description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage", + "type": "string" + }, "title": { "type": "string" }, @@ -4792,6 +4785,10 @@ "parentUid": { "type": "string" }, + "repository": { + "description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage", + "type": "string" + }, "title": { "type": "string" }, @@ -5062,6 +5059,10 @@ "isStarred": { "type": "boolean" }, + "orgId": { + "type": "integer", + "format": "int64" + }, "permanentlyDeleteDate": { "type": "string", "format": "date-time" @@ -5460,7 +5461,6 @@ }, "LibraryElementDTOMetaUser": { "type": "object", - "title": "LibraryElementDTOMetaUser defines model for LibraryElementDTOMetaUser.", "properties": { "avatarUrl": { "type": "string" @@ -5739,7 +5739,6 @@ }, "NavbarPreference": { "type": "object", - "title": "NavbarPreference defines model for NavbarPreference.", "properties": { "bookmarkUrls": { "type": "array", @@ -6249,7 +6248,7 @@ "$ref": "#/definitions/QueryHistoryPreference" }, "theme": { - "description": "Theme light, dark, empty is default", + "description": "light, dark, empty is default", "type": "string" }, "timezone": { @@ -6257,7 +6256,7 @@ "type": "string" }, "weekStart": { - "description": "WeekStart day of the week (sunday, monday, etc)", + "description": "day of the week (sunday, monday, etc)", "type": "string" } } @@ -6450,10 +6449,9 @@ }, "QueryHistoryPreference": { "type": "object", - "title": "QueryHistoryPreference defines model for QueryHistoryPreference.", "properties": { "homeTab": { - "description": "HomeTab one of: '' | 'query' | 'starred';", + "description": "one of: '' | 'query' | 'starred';", "type": "string" } } diff --git a/public/api-merged.json b/public/api-merged.json index f2d77b80ed8..8a09345e112 100644 --- a/public/api-merged.json +++ b/public/api-merged.json @@ -15477,6 +15477,10 @@ "$ref": "#/definitions/Folder" } }, + "repository": { + "description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage", + "type": "string" + }, "title": { "type": "string" }, @@ -15509,6 +15513,10 @@ "parentUid": { "type": "string" }, + "repository": { + "description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage", + "type": "string" + }, "title": { "type": "string" }, diff --git a/public/openapi3.json b/public/openapi3.json index b8c9a75b21d..cd2830849b6 100644 --- a/public/openapi3.json +++ b/public/openapi3.json @@ -5551,6 +5551,10 @@ }, "type": "array" }, + "repository": { + "description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage", + "type": "string" + }, "title": { "type": "string" }, @@ -5583,6 +5587,10 @@ "parentUid": { "type": "string" }, + "repository": { + "description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage", + "type": "string" + }, "title": { "type": "string" },