feat: add new Folder table migration & define nested folder interfaces (#56882)

* feat: add new Folder table migration

Add a new folder table to support the Nested Folders feature.

https://github.com/grafana/grafana/issues/56880

* register nested folders feature flag (unused)

* feat: nested folder service (experiment)

This commit adds a NestedFolderSvc interface and stubbed out implementation as an alternative to the existing folder service. This is an experimental feature to try out different methods for backwards compatibility and parallelization, so that Grafana can continue to store folders in the existing (non-nested) manner while also using the new nested folder service.

Eventually the new service will (hopefully) become _the_ service, at which point the legacy service can be deprecated (or remain, with the new service methods replacing the original. whatever makes sense at the time).


* nested folders: don't run the new migration

This commit removes the nested folder migration from the list of active migrations so we can merge this branch and continue development without impacting Grafana instances built off main.

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -79,4 +79,5 @@ export interface FeatureToggles {
   showDashboardValidationWarnings?: boolean;
   mysqlAnsiQuotes?: boolean;
   accessControlOnCall?: boolean;
+  nestedFolders?: boolean;
 }

pkg/services/featuremgmt/registry.go

@@ -346,5 +346,11 @@ var (
 			State:           FeatureStateAlpha,
 			RequiresDevMode: true,
 		},
+		{
+			Name:            "nestedFolders",
+			Description:     "Enable folder nesting",
+			State:           FeatureStateAlpha,
+			RequiresDevMode: true,
+		},
 	}
 )

pkg/services/featuremgmt/toggles_gen.go

@@ -258,4 +258,8 @@ const (
 	// FlagAccessControlOnCall
 	// Access control primitives for OnCall
 	FlagAccessControlOnCall = "accessControlOnCall"
+
+	// FlagNestedFolders
+	// Enable folder nesting
+	FlagNestedFolders = "nestedFolders"
 )

pkg/services/featuremgmt/toggles_gen_test.go

@@ -11,8 +11,9 @@ import (
 	"unicode"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/grafana/grafana/pkg/services/featuremgmt/strcase"
 	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/services/featuremgmt/strcase"
 )
 
 func TestFeatureToggleFiles(t *testing.T) {

pkg/services/folder/folder.go

@@ -1,20 +0,0 @@
-package folder
-
-import (
-	"context"
-
-	"github.com/grafana/grafana/pkg/models"
-	"github.com/grafana/grafana/pkg/services/user"
-)
-
-//go:generate mockery --name Service --structname FakeService --inpackage --filename foldertest/folder_service_mock.go
-type Service interface {
-	GetFolders(ctx context.Context, user *user.SignedInUser, orgID int64, limit int64, page int64) ([]*models.Folder, error)
-	GetFolderByID(ctx context.Context, user *user.SignedInUser, id int64, orgID int64) (*models.Folder, error)
-	GetFolderByUID(ctx context.Context, user *user.SignedInUser, orgID int64, uid string) (*models.Folder, error)
-	GetFolderByTitle(ctx context.Context, user *user.SignedInUser, orgID int64, title string) (*models.Folder, error)
-	CreateFolder(ctx context.Context, user *user.SignedInUser, orgID int64, title, uid string) (*models.Folder, error)
-	UpdateFolder(ctx context.Context, user *user.SignedInUser, orgID int64, existingUid string, cmd *models.UpdateFolderCommand) error
-	DeleteFolder(ctx context.Context, user *user.SignedInUser, orgID int64, uid string, forceDeleteRules bool) (*models.Folder, error)
-	MakeUserAdmin(ctx context.Context, orgID int64, userID, folderID int64, setViewAndEditPermissions bool) error
-}

pkg/services/folder/folderimpl/folder.go

@@ -90,6 +90,7 @@ func (s *Service) GetFolderByID(ctx context.Context, user *user.SignedInUser, id
 	if id == 0 {
 		return &models.Folder{Id: id, Title: "General"}, nil
 	}
+
 	dashFolder, err := s.dashboardStore.GetFolderByID(ctx, orgID, id)
 	if err != nil {
 		return nil, err

pkg/services/folder/folderimpl/store.go

@@ -0,0 +1,53 @@
+package folderimpl
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/infra/db"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/services/folder"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+type store struct {
+	db  db.DB
+	log log.Logger
+	cfg *setting.Cfg
+	fm  featuremgmt.FeatureManager
+}
+
+// store implements the folder.Store interface.
+var _ folder.Store = (*store)(nil)
+
+func ProvideStore(db db.DB, cfg *setting.Cfg, features featuremgmt.FeatureManager) *store {
+	return &store{db: db, log: log.New("folder-store"), cfg: cfg, fm: features}
+}
+
+func (s *store) Create(ctx context.Context, cmd *folder.CreateFolderCommand) (*folder.Folder, error) {
+	panic("not implemented")
+}
+
+func (s *store) Delete(ctx context.Context, uid string, orgID int64) error {
+	panic("not implemented")
+}
+
+func (s *store) Update(ctx context.Context, cmd *folder.UpdateFolderCommand) (*folder.Folder, error) {
+	panic("not implemented")
+}
+
+func (s *store) Move(ctx context.Context, cmd *folder.MoveFolderCommand) (*folder.Folder, error) {
+	panic("not implemented")
+}
+
+func (s *store) Get(ctx context.Context, cmd *folder.GetFolderCommand) (*folder.Folder, error) {
+	panic("not implemented")
+}
+
+func (s *store) GetParents(ctx context.Context, cmd *folder.GetParentsCommand) ([]*folder.Folder, error) {
+	panic("not implemented")
+}
+
+func (s *store) GetChildren(ctx context.Context, cmd *folder.GetTreeCommand) ([]*folder.Folder, error) {
+	panic("not implemented")
+}

pkg/services/folder/folderimpl/store_test.go

@@ -0,0 +1,21 @@
+package folderimpl
+
+import "testing"
+
+func TestCreate(t *testing.T) {}
+
+func TestDelete(t *testing.T) {}
+
+func TestUpdate(t *testing.T) {}
+
+func TestMove(t *testing.T) {}
+
+func TestGet(t *testing.T) {}
+
+func TestGetParent(t *testing.T) {}
+
+func TestGetParents(t *testing.T) {}
+
+func TestGetChildren(t *testing.T) {}
+
+func TestGetDescendents(t *testing.T) {}

pkg/services/folder/foldertest/foldertest.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/folder"
 	"github.com/grafana/grafana/pkg/services/user"
 )
 
@@ -13,6 +14,8 @@ type FakeService struct {
 	ExpectedError   error
 }
 
+var _ folder.Service = (*FakeService)(nil)
+
 func (s *FakeService) GetFolders(ctx context.Context, user *user.SignedInUser, orgID int64, limit int64, page int64) ([]*models.Folder, error) {
 	return s.ExpectedFolders, s.ExpectedError
 }
@@ -38,3 +41,37 @@ func (s *FakeService) DeleteFolder(ctx context.Context, user *user.SignedInUser,
 func (s *FakeService) MakeUserAdmin(ctx context.Context, orgID int64, userID, folderID int64, setViewAndEditPermissions bool) error {
 	return s.ExpectedError
 }
+
+func (s *FakeService) GetParents(ctx context.Context, orgID int64, folderUID string) ([]*folder.Folder, error) {
+	return modelsToFolders(s.ExpectedFolders), s.ExpectedError
+}
+
+func (s *FakeService) GetTree(ctx context.Context, orgID int64, folderUID string, depth int64) (map[string][]*folder.Folder, error) {
+	ret := make(map[string][]*folder.Folder)
+	ret[folderUID] = modelsToFolders(s.ExpectedFolders)
+	return ret, s.ExpectedError
+}
+
+// temporary helper until all Folder service methods are updated to use
+// folder.Folder instead of model.Folder
+func modelsToFolders(m []*models.Folder) []*folder.Folder {
+	if m == nil {
+		return nil
+	}
+	ret := make([]*folder.Folder, len(m))
+	for i, f := range m {
+		ret[i] = &folder.Folder{
+			ID:          f.Id,
+			UID:         f.Uid,
+			Title:       f.Title,
+			Description: "", // model.Folder does not have a description
+			URL:         f.Url,
+			Created:     f.Created,
+			CreatedBy:   f.CreatedBy,
+			Updated:     f.Updated,
+			UpdatedBy:   f.UpdatedBy,
+			HasACL:      f.HasACL,
+		}
+	}
+	return ret
+}

pkg/services/folder/foldertest/store.go

@@ -0,0 +1,43 @@
+package foldertest
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/services/folder"
+)
+
+type FakeStore struct {
+	ExpectedFolders []*folder.Folder
+	ExpectedFolder  *folder.Folder
+	ExpectedError   error
+}
+
+var _ folder.Store = (*FakeStore)(nil)
+
+func (f *FakeStore) Create(ctx context.Context, cmd *folder.CreateFolderCommand) (*folder.Folder, error) {
+	return f.ExpectedFolder, f.ExpectedError
+}
+
+func (f *FakeStore) Delete(ctx context.Context, uid string, orgID int64) error {
+	return f.ExpectedError
+}
+
+func (f *FakeStore) Update(ctx context.Context, cmd *folder.UpdateFolderCommand) (*folder.Folder, error) {
+	return f.ExpectedFolder, f.ExpectedError
+}
+
+func (f *FakeStore) Move(ctx context.Context, cmd *folder.MoveFolderCommand) (*folder.Folder, error) {
+	return f.ExpectedFolder, f.ExpectedError
+}
+
+func (f *FakeStore) Get(ctx context.Context, cmd *folder.GetFolderCommand) (*folder.Folder, error) {
+	return f.ExpectedFolder, f.ExpectedError
+}
+
+func (f *FakeStore) GetParents(ctx context.Context, cmd *folder.GetParentsCommand) ([]*folder.Folder, error) {
+	return f.ExpectedFolders, f.ExpectedError
+}
+
+func (f *FakeStore) GetChildren(ctx context.Context, cmd *folder.GetTreeCommand) ([]*folder.Folder, error) {
+	return f.ExpectedFolders, f.ExpectedError
+}

pkg/services/folder/model.go

@@ -0,0 +1,100 @@
+package folder
+
+import (
+	"time"
+)
+
+const (
+	GeneralFolderUID     = "general"
+	MaxNestedFolderDepth = 8
+)
+
+type Folder struct {
+	ID          int64
+	OrgID       int64
+	UID         string
+	ParentUID   string
+	Title       string
+	Description string
+
+	// TODO: is URL relevant for folders?
+	URL string
+
+	Created time.Time
+	Updated time.Time
+
+	// TODO: are these next three relevant for folders?
+	UpdatedBy int64
+	CreatedBy int64
+	HasACL    bool
+}
+
+// NewFolder tales a title and returns a Folder with the Created and Updated
+// fields set to the current time.
+func NewFolder(title string, description string) *Folder {
+	return &Folder{
+		Title:       title,
+		Description: description,
+		Created:     time.Now(),
+		Updated:     time.Now(),
+	}
+}
+
+// CreateFolderCommand captures the information required by the folder service
+// to create a folder.
+type CreateFolderCommand struct {
+	UID         string `json:"uid"`
+	Title       string `json:"title"`
+	Description string `json:"description"`
+	ParentUID   string `json:"parent_uid"`
+}
+
+// UpdateFolderCommand captures the information required by the folder service
+// to update a folder. Use Move to update a folder's parent folder.
+type UpdateFolderCommand struct {
+	Folder         *Folder `json:"folder"` // The extant folder
+	NewUID         *string `json:"uid"`
+	NewTitle       *string `json:"title"`
+	NewDescription *string `json:"description"`
+}
+
+// MoveFolderCommand captures the information required by the folder service
+// to move a folder.
+type MoveFolderCommand struct {
+	UID          string `json:"uid"`
+	NewParentUID string `json:"new_parent_uid"`
+}
+
+// DeleteFolderCommand captures the information required by the folder service
+// to delete a folder.
+type DeleteFolderCommand struct {
+	UID string `json:"uid"`
+}
+
+// GetFolderCommand is used for all folder Get requests. Only one of UID, ID, or
+// Title should be set; if multilpe fields are set by the caller the dashboard
+// service will select the field with the most specificity, in order: ID, UID,
+// Title.
+type GetFolderCommand struct {
+	UID   *string
+	ID    *int
+	Title *string
+}
+
+// GetParentsCommand captures the information required by the folder service to
+// return a list of all parent folders of a given folder.
+type GetParentsCommand struct {
+	UID string
+}
+
+// GetTreeCommand captures the information required by the folder service to
+// return a list of child folders of the given folder.
+
+type GetTreeCommand struct {
+	UID   string
+	Depth int64
+
+	// Pagination options
+	Limit int64
+	Page  int64
+}

pkg/services/folder/service.go

@@ -0,0 +1,59 @@
+package folder
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/user"
+)
+
+type Service interface {
+	GetFolders(ctx context.Context, user *user.SignedInUser, orgID int64, limit int64, page int64) ([]*models.Folder, error)
+	GetFolderByID(ctx context.Context, user *user.SignedInUser, id int64, orgID int64) (*models.Folder, error)
+	GetFolderByUID(ctx context.Context, user *user.SignedInUser, orgID int64, uid string) (*models.Folder, error)
+	GetFolderByTitle(ctx context.Context, user *user.SignedInUser, orgID int64, title string) (*models.Folder, error)
+	CreateFolder(ctx context.Context, user *user.SignedInUser, orgID int64, title, uid string) (*models.Folder, error)
+	UpdateFolder(ctx context.Context, user *user.SignedInUser, orgID int64, existingUid string, cmd *models.UpdateFolderCommand) error
+	DeleteFolder(ctx context.Context, user *user.SignedInUser, orgID int64, uid string, forceDeleteRules bool) (*models.Folder, error)
+	MakeUserAdmin(ctx context.Context, orgID int64, userID, folderID int64, setViewAndEditPermissions bool) error
+}
+
+// NestedFolderService is the temporary interface definition for the folder
+// Service which includes any new or alternate methods. These will be collapsed
+// into a single service when the nested folder implementation is rolled out.
+// Note that the commands in this service use models from this package, while
+// the legacy FolderService uses models from the models package.
+type NestedFolderService interface {
+	// Create creates a new folder.
+	Create(ctx context.Context, cmd *CreateFolderCommand) (*Folder, error)
+
+	// Update is used to update a folder's UID, Title and Description. To change
+	// a folder's parent folder, use Move.
+	Update(ctx context.Context, cmd *UpdateFolderCommand) (*Folder, error)
+
+	// Move changes a folder's parent folder to the requested new parent.
+	Move(ctx context.Context, cmd *MoveFolderCommand) (*Folder, error)
+
+	// Delete deletes a folder. This will return an error if there are any
+	// dashboards in the folder.
+	Delete(ctx context.Context, cmd *DeleteFolderCommand) (*Folder, error)
+
+	// GetFolder takes a GetFolderCommand and returns a folder matching the
+	// request. One of ID, UID, or Title must be included. If multiple values
+	// are included in the request, Grafana will select one in order of
+	// specificity (ID, UID, Title).
+	Get(ctx context.Context, cmd *GetFolderCommand) (*Folder, error)
+
+	// GetParents returns an ordered list of parent folders for the given
+	// folder, starting with the root node and ending with the requested child
+	// node.
+	GetParents(ctx context.Context, cmd *GetParentsCommand) ([]*Folder, error)
+
+	// GetTree returns an map containing all child folders starting from the
+	// given parent folder UID and descending to the requested depth. Use the
+	// sentinel value -1 to return all child folders.
+	//
+	// The map keys are folder uids and the values are the list of child folders
+	// for that parent.
+	GetTree(ctx context.Context, cmd *GetTreeCommand) (map[string][]*Folder, error)
+}

pkg/services/folder/store.go

@@ -0,0 +1,29 @@
+package folder
+
+import "context"
+
+// Store is the interface which a folder store must implement.
+type Store interface {
+	// Create creates a folder and returns the newly-created folder.
+	Create(ctx context.Context, cmd *CreateFolderCommand) (*Folder, error)
+
+	// Delete deletes a folder from the folder store.
+	Delete(ctx context.Context, uid string, orgID int64) error
+
+	// Update updates the given folder's UID, Title, and Description.
+	// Use Move to change a dashboard's parent ID.
+	Update(ctx context.Context, cmd *UpdateFolderCommand) (*Folder, error)
+
+	// Move changes the given folder's parent folder uid and applies any necessary permissions changes.
+	Move(ctx context.Context, cmd *MoveFolderCommand) (*Folder, error)
+
+	// Get returns a folder.
+	Get(ctx context.Context, cmd *GetFolderCommand) (*Folder, error)
+
+	// GetParents returns an ordered list of parent folder of the given folder.
+	GetParents(ctx context.Context, cmd *GetParentsCommand) ([]*Folder, error)
+
+	// GetChildren returns the set of immediate children folders (depth=1) of the
+	// given folder.
+	GetChildren(ctx context.Context, cmd *GetTreeCommand) ([]*Folder, error)
+}

pkg/services/sqlstore/migrations/folder_mig.go

@@ -0,0 +1,39 @@
+package migrations
+
+import (
+	"github.com/grafana/grafana/pkg/services/folder"
+	"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
+)
+
+// nolint:unused // this is temporarily unused during feature development
+func addFolderMigrations(mg *migrator.Migrator) {
+	mg.AddMigration("create folder table", migrator.NewAddTableMigration(folderv1()))
+
+	// copy any existing folders in the dashboard table into the new folder
+	// table. The *legacy* parent folder ID, stored as folder_id  in the
+	// dashboard table, is always going to be "0" so it is safe to convert to a parent UID.
+	mg.AddMigration("copy existing folders from dashboard table", migrator.NewRawSQLMigration(
+		"INSERT INTO folder (id, uid, org_id, title, parent_uid, created, updated) SELECT id, uid, org_id, title, folder_id, created, updated FROM dashboard WHERE is_folder = 1;",
+	).Postgres("INSERT INTO folder (id, uid, org_id, title, parent_uid, created, updated) SELECT id, uid, org_id, title, folder_id, created, updated FROM dashboard WHERE is_folder = true;"))
+
+	mg.AddMigration("Add index for parent_uid", migrator.NewAddIndexMigration(folderv1(), &migrator.Index{
+		Cols: []string{"parent_uid", "org_id"},
+	}))
+}
+
+// nolint:unused // this is temporarily unused during feature development
+func folderv1() migrator.Table {
+	return migrator.Table{
+		Name: "folder",
+		Columns: []*migrator.Column{
+			{Name: "id", Type: migrator.DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
+			{Name: "uid", Type: migrator.DB_NVarchar, Length: 40},
+			{Name: "org_id", Type: migrator.DB_BigInt, Nullable: false},
+			{Name: "title", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "description", Type: migrator.DB_NVarchar, Length: 255, Nullable: true},
+			{Name: "parent_uid", Type: migrator.DB_NVarchar, Length: 40, Default: folder.GeneralFolderUID},
+			{Name: "created", Type: migrator.DB_DateTime, Nullable: false},
+			{Name: "updated", Type: migrator.DB_DateTime, Nullable: false},
+		},
+	}
+}

pkg/services/sqlstore/migrations/migrations.go

@@ -99,6 +99,13 @@ func (*OSSMigrations) AddMigration(mg *Migrator) {
 	accesscontrol.AddManagedFolderAlertActionsRepeatMigration(mg)
 	accesscontrol.AddAdminOnlyMigration(mg)
 	accesscontrol.AddSeedAssignmentMigrations(mg)
+
+	// TODO: This migration will be enabled later in the nested folder feature
+	// implementation process. It is on hold so we can continue working on the
+	// store implementation without impacting any grafana instances built off
+	// main.
+	//
+	// addFolderMigrations(mg)
 }
 
 func addMigrationLogMigrations(mg *Migrator) {