Folders: Convert between unstructured and legacy (#99504)

pkg/api/dtos/folder.go

@@ -28,6 +28,10 @@ type Folder struct {
 	ParentUID string `json:"parentUid,omitempty"`
 	// the parent folders starting from the root going down
 	Parents []Folder `json:"parents,omitempty"`
+
+	// When the folder belongs to a repository
+	// NOTE: this is only populated when folders are managed by unified storage
+	Repository string `json:"repository,omitempty"`
 }
 
 type FolderSearchHit struct {
@@ -35,4 +39,8 @@ type FolderSearchHit struct {
 	UID       string `json:"uid" xorm:"uid"`
 	Title     string `json:"title"`
 	ParentUID string `json:"parentUid,omitempty"`
+
+	// When the folder belongs to a repository
+	// NOTE: this is only populated when folders are managed by unified storage
+	Repository string `json:"repository,omitempty"`
 }

pkg/api/folder.go

@@ -98,6 +98,7 @@ func (hs *HTTPServer) GetFolders(c *contextmodel.ReqContext) response.Response {
 				UID:        f.UID,
 				Title:      f.Title,
 				ParentUID:  f.ParentUID,
+				Repository: f.Repository,
 			})
 			metrics.MFolderIDsAPICount.WithLabelValues(metrics.GetFolders).Inc()
 		}
@@ -425,6 +426,7 @@ func (hs *HTTPServer) newToFolderDto(c *contextmodel.ReqContext, f *folder.Folde
 			Version:       f.Version,
 			AccessControl: acMetadata,
 			ParentUID:     f.ParentUID,
+			Repository:    f.Repository,
 		}, nil
 	}
 

pkg/registry/apis/folders/conversions.go

@@ -2,11 +2,11 @@ package folders
 
 import (
 	"fmt"
-	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
+	authlib "github.com/grafana/authlib/types"
 	"github.com/grafana/grafana/pkg/apimachinery/utils"
 	"github.com/grafana/grafana/pkg/apis/folder/v0alpha1"
 	"github.com/grafana/grafana/pkg/infra/slugify"
@@ -26,98 +26,65 @@ func LegacyCreateCommandToUnstructured(cmd *folder.CreateFolderCommand) (*unstru
 			},
 		},
 	}
-	// #TODO: let's see if we need to set the json field to "-"
+
+	meta, err := utils.MetaAccessor(obj)
+	if err != nil {
+		return nil, err
+	}
+
 	if cmd.UID == "" {
 		cmd.UID = util.GenerateShortUID()
 	}
-	obj.SetName(cmd.UID)
+	meta.SetName(cmd.UID)
-
+	meta.SetFolder(cmd.ParentUID)
-	if err := setParentUID(obj, cmd.ParentUID); err != nil {
-		return &unstructured.Unstructured{}, err
-	}
 
 	return obj, nil
 }
 
-func LegacyUpdateCommandToUnstructured(obj *unstructured.Unstructured, cmd *folder.UpdateFolderCommand) (*unstructured.Unstructured, error) {
+func UnstructuredToLegacyFolder(item *unstructured.Unstructured) (*folder.Folder, error) {
-	spec, ok := obj.Object["spec"].(map[string]any)
+	meta, err := utils.MetaAccessor(item)
-	if !ok {
-		return &unstructured.Unstructured{}, fmt.Errorf("could not convert object to folder")
-	}
-	if cmd.NewTitle != nil {
-		spec["title"] = cmd.NewTitle
-	}
-	if cmd.NewDescription != nil {
-		spec["description"] = cmd.NewDescription
-	}
-	if cmd.NewParentUID != nil {
-		if err := setParentUID(obj, *cmd.NewParentUID); err != nil {
-			return &unstructured.Unstructured{}, err
-		}
-	}
-
-	return obj, nil
-}
-
-func LegacyMoveCommandToUnstructured(obj *unstructured.Unstructured, cmd folder.MoveFolderCommand) (*unstructured.Unstructured, error) {
-	if err := setParentUID(obj, cmd.NewParentUID); err != nil {
-		return &unstructured.Unstructured{}, err
-	}
-
-	return obj, nil
-}
-
-func UnstructuredToLegacyFolder(item unstructured.Unstructured, orgID int64) (*folder.Folder, string) {
-	// #TODO reduce duplication of the different conversion functions
-	spec := item.Object["spec"].(map[string]any)
-	uid := item.GetName()
-	title := spec["title"].(string)
-
-	meta, err := utils.MetaAccessor(&item)
 	if err != nil {
-		return nil, ""
+		return nil, err
 	}
 
-	id := meta.GetDeprecatedInternalID() // nolint:staticcheck
+	info, _ := authlib.ParseNamespace(meta.GetNamespace())
-
+	if info.OrgID < 0 {
-	created, err := getCreated(meta)
+		info.OrgID = 1 // This resolves all test cases that assume org 1
-	if err != nil {
-		return nil, ""
 	}
 
-	// avoid panic
+	title, _, _ := unstructured.NestedString(item.Object, "spec", "title")
-	var createdTime time.Time
+	description, _, _ := unstructured.NestedString(item.Object, "spec", "description")
-	if created != nil {
+
-		// #TODO Fix this time format. The legacy time format seems to be along the lines of time.Now()
+	uid := meta.GetName()
-		// which includes a part that represents a fraction of a second. Format should be "2024-09-12T15:37:41.09466+02:00"
+	url := ""
-		createdTime = (*created).UTC()
+	if uid != folder.RootFolder.UID {
+		slug := slugify.Slugify(title)
+		url = dashboards.GetFolderURL(uid, slug)
 	}
 
-	url := getURL(meta, title)
+	created := meta.GetCreationTimestamp().Time.UTC()
-
+	updated, _ := meta.GetUpdatedTimestamp()
-	// RootFolder does not have URL
+	if updated == nil {
-	if uid == folder.RootFolder.UID {
+		updated = &created
-		url = ""
+	} else {
+		tmp := updated.UTC()
+		updated = &tmp
 	}
 
-	f := &folder.Folder{
+	return &folder.Folder{
 		UID:         uid,
 		Title:       title,
-		ID:        id,
+		Description: description,
+		ID:          meta.GetDeprecatedInternalID(), // nolint:staticcheck
 		ParentUID:   meta.GetFolder(),
-		// #TODO add created by field if necessary
+		Version:     int(meta.GetGeneration()),
+		Repository:  meta.GetRepositoryName(),
+
 		URL:     url,
-		// #TODO get Created in format "2024-09-12T15:37:41.09466+02:00"
+		Created: created,
-		Created: createdTime,
+		Updated: *updated,
-		// #TODO figure out whether we want to set "updated" and "updated by". Could replace with
+		OrgID:   info.OrgID,
-		// meta.GetUpdatedTimestamp() but it currently gets overwritten in prepareObjectForStorage().
+	}, nil
-		Updated: createdTime,
-		OrgID:   orgID,
-	}
-	// CreatedBy needs to be returned separately because it's the user UID (string) but
-	// folder.Folder expects user ID (int64).
-	return f, meta.GetCreatedBy()
-	// #TODO figure out about adding version, parents, orgID fields
 }
 
 func LegacyFolderToUnstructured(v *folder.Folder, namespacer request.NamespaceMapper) (*v0alpha1.Folder, error) {
@@ -164,23 +131,3 @@ func convertToK8sResource(v *folder.Folder, namespacer request.NamespaceMapper)
 	f.UID = gapiutil.CalculateClusterWideUID(f)
 	return f, nil
 }
-
-func setParentUID(u *unstructured.Unstructured, parentUid string) error {
-	meta, err := utils.MetaAccessor(u)
-	if err != nil {
-		return err
-	}
-	meta.SetFolder(parentUid)
-	return nil
-}
-
-func getURL(meta utils.GrafanaMetaAccessor, title string) string {
-	slug := slugify.Slugify(title)
-	uid := meta.GetName()
-	return dashboards.GetFolderURL(uid, slug)
-}
-
-func getCreated(meta utils.GrafanaMetaAccessor) (*time.Time, error) {
-	created := meta.GetCreationTimestamp().Time
-	return &created, nil
-}

pkg/registry/apis/folders/conversions_test.go

@@ -0,0 +1,62 @@
+package folders
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	"github.com/grafana/grafana/pkg/services/folder"
+)
+
+func TestFolderConversions(t *testing.T) {
+	input := &unstructured.Unstructured{}
+	err := input.UnmarshalJSON([]byte(`{
+      "kind": "Folder",
+      "apiVersion": "folder.grafana.app/v0alpha1",
+      "metadata": {
+        "name": "be79sztagf20wd",
+        "namespace": "default",
+        "uid": "wfi3RARqQREzEKtUJCWurWevwbQ7i9ii0cA7JUIbMtEX",
+        "resourceVersion": "1734509107000",
+        "creationTimestamp": "2022-12-02T02:02:02Z",
+				"generation": 4,
+        "labels": {
+          "grafana.app/deprecatedInternalID": "234"
+        },
+        "annotations": {
+          "grafana.app/folder": "parent-folder-name",
+          "grafana.app/updatedTimestamp": "2022-12-02T07:02:02Z",
+          "grafana.app/repoName": "example-repo",
+          "grafana.app/createdBy": "user:abc",
+          "grafana.app/updatedBy": "service:xyz"
+        }
+      },
+      "spec": {
+        "title": "test folder",
+        "description": "Something set in the file"
+      }
+    }`))
+	require.NoError(t, err)
+
+	created, err := time.Parse(time.RFC3339, "2022-12-02T02:02:02Z")
+	created = created.UTC()
+	require.NoError(t, err)
+
+	converted, err := UnstructuredToLegacyFolder(input)
+	require.NoError(t, err)
+	require.Equal(t, folder.Folder{
+		ID:          234,
+		OrgID:       1,
+		Version:     4,
+		UID:         "be79sztagf20wd",
+		ParentUID:   "parent-folder-name",
+		Title:       "test folder",
+		Description: "Something set in the file",
+		URL:         "/dashboards/f/be79sztagf20wd/test-folder",
+		Repository:  "example-repo",
+		Created:     created,
+		Updated:     created.Add(time.Hour * 5),
+	}, *converted)
+}

pkg/services/folder/folderimpl/unifiedstore.go

@@ -6,13 +6,15 @@ import (
 	"strings"
 	"time"
 
-	"github.com/grafana/grafana/pkg/apimachinery/identity"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	k8sUser "k8s.io/apiserver/pkg/authentication/user"
 	k8sRequest "k8s.io/apiserver/pkg/endpoints/request"
 
+	"github.com/grafana/grafana/pkg/apimachinery/identity"
+	"github.com/grafana/grafana/pkg/apimachinery/utils"
+
 	"github.com/grafana/grafana/pkg/apis/folder/v0alpha1"
 	"github.com/grafana/grafana/pkg/infra/log"
 	internalfolders "github.com/grafana/grafana/pkg/registry/apis/folders"
@@ -58,7 +60,7 @@ func (ss *FolderUnifiedStoreImpl) Create(ctx context.Context, cmd folder.CreateF
 		return nil, err
 	}
 
-	folder, _ := internalfolders.UnstructuredToLegacyFolder(*out, cmd.SignedInUser.GetOrgID())
+	folder, err := internalfolders.UnstructuredToLegacyFolder(out)
 	if err != nil {
 		return nil, err
 	}
@@ -106,23 +108,34 @@ func (ss *FolderUnifiedStoreImpl) Update(ctx context.Context, cmd folder.UpdateF
 	if err != nil {
 		return nil, err
 	}
+	updated := obj.DeepCopy()
 
-	updated, err := internalfolders.LegacyUpdateCommandToUnstructured(obj, &cmd)
+	if cmd.NewTitle != nil {
+		err = unstructured.SetNestedField(updated.Object, *cmd.NewTitle, "spec", "title")
 		if err != nil {
 			return nil, err
 		}
+	}
+	if cmd.NewDescription != nil {
+		err = unstructured.SetNestedField(updated.Object, *cmd.NewDescription, "spec", "description")
+		if err != nil {
+			return nil, err
+		}
+	}
+	if cmd.NewParentUID != nil {
+		meta, err := utils.MetaAccessor(updated)
+		if err != nil {
+			return nil, err
+		}
+		meta.SetFolder(*cmd.NewParentUID)
+	}
 
 	out, err := client.Update(ctx, updated, v1.UpdateOptions{})
 	if err != nil {
 		return nil, err
 	}
 
-	folder, _ := internalfolders.UnstructuredToLegacyFolder(*out, cmd.SignedInUser.GetOrgID())
+	return internalfolders.UnstructuredToLegacyFolder(out)
-	if err != nil {
-		return nil, err
-	}
-
-	return folder, err
 }
 
 // If WithFullpath is true it computes also the full path of a folder.
@@ -164,8 +177,8 @@ func (ss *FolderUnifiedStoreImpl) Get(ctx context.Context, q folder.GetFolderQue
 	} else if err != nil || out == nil {
 		return nil, dashboards.ErrFolderNotFound
 	}
-	dashFolder, _ := internalfolders.UnstructuredToLegacyFolder(*out, q.SignedInUser.GetOrgID())
+
-	return dashFolder, nil
+	return internalfolders.UnstructuredToLegacyFolder(out)
 }
 
 func (ss *FolderUnifiedStoreImpl) GetParents(ctx context.Context, q folder.GetParentsQuery) ([]*folder.Folder, error) {
@@ -193,7 +206,7 @@ func (ss *FolderUnifiedStoreImpl) GetParents(ctx context.Context, q folder.GetPa
 			return nil, err
 		}
 
-		folder, _ := internalfolders.UnstructuredToLegacyFolder(*out, q.OrgID)
+		folder, err := internalfolders.UnstructuredToLegacyFolder(out)
 		if err != nil {
 			return nil, err
 		}
@@ -232,9 +245,9 @@ func (ss *FolderUnifiedStoreImpl) GetChildren(ctx context.Context, q folder.GetC
 	hits := make([]*folder.Folder, 0)
 	for _, item := range out.Items {
 		// convert item to legacy folder format
-		f, _ := internalfolders.UnstructuredToLegacyFolder(item, q.OrgID)
+		f, err := internalfolders.UnstructuredToLegacyFolder(&item)
 		if f == nil {
-			return nil, fmt.Errorf("unable covert unstructured item to legacy folder")
+			return nil, fmt.Errorf("unable covert unstructured item to legacy folder %w", err)
 		}
 
 		// it we are at root level, skip subfolder
@@ -332,9 +345,9 @@ func (ss *FolderUnifiedStoreImpl) GetFolders(ctx context.Context, q folder.GetFo
 	m := map[string]*folder.Folder{}
 	for _, item := range out.Items {
 		// convert item to legacy folder format
-		f, _ := internalfolders.UnstructuredToLegacyFolder(item, q.SignedInUser.GetOrgID())
+		f, err := internalfolders.UnstructuredToLegacyFolder(&item)
 		if f == nil {
-			return nil, fmt.Errorf("unable covert unstructured item to legacy folder")
+			return nil, fmt.Errorf("unable covert unstructured item to legacy folder %w", err)
 		}
 
 		m[f.UID] = f
@@ -392,9 +405,9 @@ func (ss *FolderUnifiedStoreImpl) GetDescendants(ctx context.Context, orgID int6
 	nodes := map[string]*folder.Folder{}
 	for _, item := range out.Items {
 		// convert item to legacy folder format
-		f, _ := internalfolders.UnstructuredToLegacyFolder(item, orgID)
+		f, err := internalfolders.UnstructuredToLegacyFolder(&item)
 		if f == nil {
-			return nil, fmt.Errorf("unable covert unstructured item to legacy folder")
+			return nil, fmt.Errorf("unable covert unstructured item to legacy folder %w", err)
 		}
 
 		nodes[f.UID] = f

pkg/services/folder/model.go

@@ -53,6 +53,11 @@ type Folder struct {
 	HasACL       bool
 	Fullpath     string `xorm:"fullpath"`
 	FullpathUIDs string `xorm:"fullpath_uids"`
+
+	// When the folder belongs to a repository
+	// NOTE: this is only populated when folders are managed by unified storage
+	// This is not ever used by xorm, but the translation functions flow through this type
+	Repository string `json:"repository,omitempty"`
 }
 
 var GeneralFolder = Folder{ID: 0, Title: "General"}

public/api-enterprise-spec.json

@@ -2837,7 +2837,6 @@
     },
     "AnnotationPanelFilter": {
       "type": "object",
-      "title": "AnnotationPanelFilter defines model for AnnotationPanelFilter.",
       "properties": {
         "exclude": {
           "description": "Should the specified panels be included or excluded",
@@ -2848,7 +2847,7 @@
           "type": "array",
           "items": {
             "type": "integer",
-            "format": "int64"
+            "format": "uint8"
           }
         }
       }
@@ -2871,7 +2870,7 @@
         "builtIn": {
           "description": "Set to 1 for the standard annotation query all dashboards have by default.",
           "type": "number",
-          "format": "float"
+          "format": "double"
         },
         "datasource": {
           "$ref": "#/definitions/DataSourceRef"
@@ -3438,20 +3437,10 @@
     },
     "CookiePreferences": {
       "type": "object",
-      "title": "CookiePreferences defines model for CookiePreferences.",
       "properties": {
-        "analytics": {
+        "analytics": {},
-          "type": "object",
+        "functional": {},
-          "additionalProperties": {}
+        "performance": {}
-        },
-        "functional": {
-          "type": "object",
-          "additionalProperties": {}
-        },
-        "performance": {
-          "type": "object",
-          "additionalProperties": {}
-        }
       }
     },
     "CookieType": {
@@ -4760,6 +4749,10 @@
             "$ref": "#/definitions/Folder"
           }
         },
+        "repository": {
+          "description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage",
+          "type": "string"
+        },
         "title": {
           "type": "string"
         },
@@ -4792,6 +4785,10 @@
         "parentUid": {
           "type": "string"
         },
+        "repository": {
+          "description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage",
+          "type": "string"
+        },
         "title": {
           "type": "string"
         },
@@ -5062,6 +5059,10 @@
         "isStarred": {
           "type": "boolean"
         },
+        "orgId": {
+          "type": "integer",
+          "format": "int64"
+        },
         "permanentlyDeleteDate": {
           "type": "string",
           "format": "date-time"
@@ -5460,7 +5461,6 @@
     },
     "LibraryElementDTOMetaUser": {
       "type": "object",
-      "title": "LibraryElementDTOMetaUser defines model for LibraryElementDTOMetaUser.",
       "properties": {
         "avatarUrl": {
           "type": "string"
@@ -5739,7 +5739,6 @@
     },
     "NavbarPreference": {
       "type": "object",
-      "title": "NavbarPreference defines model for NavbarPreference.",
       "properties": {
         "bookmarkUrls": {
           "type": "array",
@@ -6249,7 +6248,7 @@
           "$ref": "#/definitions/QueryHistoryPreference"
         },
         "theme": {
-          "description": "Theme light, dark, empty is default",
+          "description": "light, dark, empty is default",
           "type": "string"
         },
         "timezone": {
@@ -6257,7 +6256,7 @@
           "type": "string"
         },
         "weekStart": {
-          "description": "WeekStart day of the week (sunday, monday, etc)",
+          "description": "day of the week (sunday, monday, etc)",
           "type": "string"
         }
       }
@@ -6450,10 +6449,9 @@
     },
     "QueryHistoryPreference": {
       "type": "object",
-      "title": "QueryHistoryPreference defines model for QueryHistoryPreference.",
       "properties": {
         "homeTab": {
-          "description": "HomeTab one of: '' | 'query' | 'starred';",
+          "description": "one of: '' | 'query' | 'starred';",
           "type": "string"
         }
       }

public/api-merged.json

@@ -15477,6 +15477,10 @@
             "$ref": "#/definitions/Folder"
           }
         },
+        "repository": {
+          "description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage",
+          "type": "string"
+        },
         "title": {
           "type": "string"
         },
@@ -15509,6 +15513,10 @@
         "parentUid": {
           "type": "string"
         },
+        "repository": {
+          "description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage",
+          "type": "string"
+        },
         "title": {
           "type": "string"
         },

public/openapi3.json

@@ -5551,6 +5551,10 @@
             },
             "type": "array"
           },
+          "repository": {
+            "description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage",
+            "type": "string"
+          },
           "title": {
             "type": "string"
           },
@@ -5583,6 +5587,10 @@
           "parentUid": {
             "type": "string"
           },
+          "repository": {
+            "description": "When the folder belongs to a repository\nNOTE: this is only populated when folders are managed by unified storage",
+            "type": "string"
+          },
           "title": {
             "type": "string"
           },