Storage: unify List queries (#46572)

* silence errors

* s3 fix - don't retrieve files with path equal to the root

* Storage: unify list queries

* Storage: add `IsFolder` method to file obj

* Storage: API consistency - always refer `File` as a pointer rather than a value

pkg/infra/filestorage/api.go

@@ -15,6 +15,7 @@ var (
 	ErrPathInvalid           = errors.New("path is invalid")
 	ErrPathEndsWithDelimiter = errors.New("path can not end with delimiter")
 	Delimiter                = "/"
+	DirectoryMimeType        = "directory"
 	multipleDelimiters       = regexp.MustCompile(`/+`)
 )
 
@@ -30,6 +31,10 @@ type File struct {
 	FileMetadata
 }
 
+func (f *File) IsFolder() bool {
+	return f.MimeType == DirectoryMimeType
+}
+
 type FileMetadata struct {
 	Name       string
 	FullPath   string
@@ -40,12 +45,6 @@ type FileMetadata struct {
 	Properties map[string]string
 }
 
-type ListFilesResponse struct {
-	Files    []FileMetadata
-	HasMore  bool
-	LastPath string
-}
-
 type Paging struct {
 	After string
 	First int
@@ -133,8 +132,17 @@ func (f *PathFilters) IsAllowed(path string) bool {
 	return false
 }
 
+type ListResponse struct {
+	Files    []*File
+	HasMore  bool
+	LastPath string
+}
+
 type ListOptions struct {
-	Recursive bool
+	Recursive    bool
+	WithFiles    bool
+	WithFolders  bool
+	WithContents bool
 	*PathFilters
 }
 
@@ -143,8 +151,8 @@ type FileStorage interface {
 	Delete(ctx context.Context, path string) error
 	Upsert(ctx context.Context, command *UpsertFileCommand) error
 
-	ListFiles(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListFilesResponse, error)
-	ListFolders(ctx context.Context, folderPath string, options *ListOptions) ([]FileMetadata, error)
+	// List lists only files without content by default
+	List(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListResponse, error)
 
 	CreateFolder(ctx context.Context, path string) error
 	DeleteFolder(ctx context.Context, path string) error

pkg/infra/filestorage/cdk_blob_filestorage.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"sort"
 	"strings"
 
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -134,145 +133,6 @@ func (c cdkBlobStorage) Upsert(ctx context.Context, command *UpsertFileCommand)
 	})
 }
 
-func (c cdkBlobStorage) listFiles(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListFilesResponse, error) {
-	iterator := c.bucket.List(&blob.ListOptions{
-		Prefix:    strings.ToLower(folderPath),
-		Delimiter: Delimiter,
-	})
-
-	recursive := options.Recursive
-
-	pageSize := paging.First
-
-	foundCursor := true
-	if paging.After != "" {
-		foundCursor = false
-	}
-
-	hasMore := true
-	files := make([]FileMetadata, 0)
-	for {
-		obj, err := iterator.Next(ctx)
-		if obj != nil && strings.HasSuffix(obj.Key, directoryMarker) {
-			continue
-		}
-
-		if errors.Is(err, io.EOF) {
-			hasMore = false
-			break
-		} else {
-			hasMore = true
-		}
-
-		if err != nil {
-			c.log.Error("Failed while iterating over files", "err", err)
-			return nil, err
-		}
-
-		if len(files) >= pageSize {
-			break
-		}
-
-		path := obj.Key
-
-		allowed := options.IsAllowed(obj.Key)
-		if obj.IsDir && recursive {
-			newPaging := &Paging{
-				First: pageSize - len(files),
-			}
-			if paging != nil {
-				newPaging.After = paging.After
-			}
-
-			resp, err := c.listFiles(ctx, path, newPaging, options)
-
-			if err != nil {
-				return nil, err
-			}
-
-			if len(files) > 0 {
-				foundCursor = true
-			}
-
-			files = append(files, resp.Files...)
-			if len(files) >= pageSize {
-				//nolint: staticcheck
-				hasMore = resp.HasMore
-			}
-		} else if !obj.IsDir && allowed {
-			if !foundCursor {
-				res := strings.Compare(obj.Key, paging.After)
-				if res < 0 {
-					continue
-				} else if res == 0 {
-					foundCursor = true
-					continue
-				} else {
-					foundCursor = true
-				}
-			}
-
-			attributes, err := c.bucket.Attributes(ctx, strings.ToLower(path))
-			if err != nil {
-				if gcerrors.Code(err) == gcerrors.NotFound {
-					attributes, err = c.bucket.Attributes(ctx, path)
-					if err != nil {
-						c.log.Error("Failed while retrieving attributes", "path", path, "err", err)
-						return nil, err
-					}
-				} else {
-					c.log.Error("Failed while retrieving attributes", "path", path, "err", err)
-					return nil, err
-				}
-			}
-
-			if attributes.ContentType == "application/x-directory; charset=UTF-8" {
-				// S3 directory representation
-				continue
-			}
-
-			if attributes.ContentType == "text/plain" && obj.Key == folderPath && attributes.Size == 0 {
-				// GCS directory representation
-				continue
-			}
-
-			var originalPath string
-			var props map[string]string
-			if attributes.Metadata != nil {
-				props = attributes.Metadata
-				if path, ok := attributes.Metadata[originalPathAttributeKey]; ok {
-					originalPath = path
-					delete(props, originalPathAttributeKey)
-				}
-			} else {
-				props = make(map[string]string)
-				originalPath = strings.TrimSuffix(path, Delimiter)
-			}
-
-			files = append(files, FileMetadata{
-				Name:       getName(originalPath),
-				FullPath:   originalPath,
-				Created:    attributes.CreateTime,
-				Properties: props,
-				Modified:   attributes.ModTime,
-				Size:       attributes.Size,
-				MimeType:   detectContentType(originalPath, attributes.ContentType),
-			})
-		}
-	}
-
-	lastPath := ""
-	if len(files) > 0 {
-		lastPath = files[len(files)-1].FullPath
-	}
-
-	return &ListFilesResponse{
-		Files:    files,
-		HasMore:  hasMore,
-		LastPath: lastPath,
-	}, nil
-}
-
 func (c cdkBlobStorage) convertFolderPathToPrefix(path string) string {
 	if path != "" && !strings.HasSuffix(path, Delimiter) {
 		return path + Delimiter
@@ -280,124 +140,6 @@ func (c cdkBlobStorage) convertFolderPathToPrefix(path string) string {
 	return path
 }
 
-func (c cdkBlobStorage) ListFiles(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListFilesResponse, error) {
-	prefix := c.convertFolderPathToPrefix(folderPath)
-	files, err := c.listFiles(ctx, prefix, paging, options)
-	return files, err
-}
-
-func (c cdkBlobStorage) listFolderPaths(ctx context.Context, parentFolderPath string, options *ListOptions) ([]string, error) {
-	iterator := c.bucket.List(&blob.ListOptions{
-		Prefix:    strings.ToLower(parentFolderPath),
-		Delimiter: Delimiter,
-	})
-
-	recursive := options.Recursive
-
-	dirPath := ""
-	dirMarkerPath := ""
-	foundPaths := make([]string, 0)
-	for {
-		obj, err := iterator.Next(ctx)
-		if errors.Is(err, io.EOF) {
-			break
-		}
-
-		if err != nil {
-			c.log.Error("Failed while iterating over files", "err", err)
-			return nil, err
-		}
-
-		if options.IsAllowed(obj.Key) {
-			if obj.IsDir && !recursive && options.IsAllowed(obj.Key) {
-				var nestedDirPath string
-				dirMPath := obj.Key + directoryMarker
-				attributes, err := c.bucket.Attributes(ctx, dirMPath)
-				if err != nil {
-					c.log.Error("Failed while retrieving attributes", "path", obj.Key, "err", err)
-				}
-
-				if attributes != nil && attributes.Metadata != nil {
-					if path, ok := attributes.Metadata[originalPathAttributeKey]; ok {
-						nestedDirPath = getParentFolderPath(path)
-					}
-				}
-
-				if nestedDirPath != "" {
-					foundPaths = append(foundPaths, nestedDirPath)
-				} else {
-					foundPaths = append(foundPaths, strings.TrimSuffix(obj.Key, Delimiter))
-				}
-			}
-
-			if dirPath == "" && !obj.IsDir {
-				dirPath = getParentFolderPath(obj.Key)
-			}
-
-			if dirMarkerPath == "" && !obj.IsDir {
-				attributes, err := c.bucket.Attributes(ctx, obj.Key)
-				if err != nil {
-					c.log.Error("Failed while retrieving attributes", "path", obj.Key, "err", err)
-					return nil, err
-				}
-
-				if attributes.Metadata != nil {
-					if path, ok := attributes.Metadata[originalPathAttributeKey]; ok {
-						dirMarkerPath = getParentFolderPath(path)
-					}
-				}
-			}
-		}
-
-		if obj.IsDir && recursive {
-			resp, err := c.listFolderPaths(ctx, obj.Key, options)
-
-			if err != nil {
-				return nil, err
-			}
-
-			if len(resp) > 0 {
-				foundPaths = append(foundPaths, resp...)
-			}
-			continue
-		}
-	}
-
-	var foundPath string
-	if dirMarkerPath != "" {
-		foundPath = dirMarkerPath
-	} else if dirPath != "" {
-		foundPath = dirPath
-	}
-
-	if foundPath != "" && options.IsAllowed(foundPath+Delimiter) {
-		foundPaths = append(foundPaths, foundPath)
-	}
-	return foundPaths, nil
-}
-
-func (c cdkBlobStorage) ListFolders(ctx context.Context, prefix string, options *ListOptions) ([]FileMetadata, error) {
-	fixedPrefix := c.convertFolderPathToPrefix(prefix)
-	foundPaths, err := c.listFolderPaths(ctx, fixedPrefix, options)
-	if err != nil {
-		return nil, err
-	}
-
-	sort.Strings(foundPaths)
-	folders := make([]FileMetadata, 0)
-
-	for _, path := range foundPaths {
-		if strings.Compare(path, fixedPrefix) > 0 {
-			folders = append(folders, FileMetadata{
-				Name:     getName(path),
-				FullPath: path,
-			})
-		}
-	}
-
-	return folders, err
-}
-
 func precedingFolders(path string) []string {
 	parts := strings.Split(path, Delimiter)
 	if len(parts) == 0 {
@@ -489,6 +231,186 @@ func (c cdkBlobStorage) DeleteFolder(ctx context.Context, folderPath string) err
 	return err
 }
 
+//nolint: gocyclo
+func (c cdkBlobStorage) list(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListResponse, error) {
+	lowerRootPath := strings.ToLower(folderPath)
+	iterators := []*blob.ListIterator{c.bucket.List(&blob.ListOptions{
+		Prefix:    lowerRootPath,
+		Delimiter: Delimiter,
+	})}
+
+	recursive := options.Recursive
+	pageSize := paging.First
+
+	foundCursor := true
+	if paging.After != "" {
+		foundCursor = false
+	}
+
+	files := make([]*File, 0)
+
+	visitedFolders := map[string]bool{}
+	visitedFolders[lowerRootPath] = true
+
+	for len(iterators) > 0 && len(files) <= pageSize {
+		obj, err := iterators[0].Next(ctx)
+		if errors.Is(err, io.EOF) {
+			iterators = iterators[1:]
+			continue
+		}
+
+		if err != nil {
+			c.log.Error("Failed while iterating over files", "err", err)
+			return nil, err
+		}
+
+		path := obj.Key
+		lowerPath := strings.ToLower(path)
+		allowed := options.IsAllowed(lowerPath)
+
+		if obj.IsDir && recursive && !visitedFolders[lowerPath] {
+			iterators = append([]*blob.ListIterator{c.bucket.List(&blob.ListOptions{
+				Prefix:    lowerPath,
+				Delimiter: Delimiter,
+			})}, iterators...)
+			visitedFolders[lowerPath] = true
+		}
+
+		if !foundCursor {
+			res := strings.Compare(strings.TrimSuffix(lowerPath, Delimiter), paging.After)
+			if res < 0 {
+				continue
+			} else if res == 0 {
+				foundCursor = true
+				continue
+			} else {
+				foundCursor = true
+			}
+		}
+
+		if obj.IsDir {
+			if options.WithFolders && allowed {
+				originalCasingPath := ""
+				dirMarkerPath := obj.Key + directoryMarker
+				attributes, err := c.bucket.Attributes(ctx, dirMarkerPath)
+				if err == nil && attributes != nil && attributes.Metadata != nil {
+					if path, ok := attributes.Metadata[originalPathAttributeKey]; ok {
+						originalCasingPath = getParentFolderPath(path)
+					}
+				}
+
+				var p string
+				if originalCasingPath != "" {
+					p = originalCasingPath
+				} else {
+					p = strings.TrimSuffix(obj.Key, Delimiter)
+				}
+
+				files = append(files, &File{
+					Contents: nil,
+					FileMetadata: FileMetadata{
+						MimeType:   DirectoryMimeType,
+						Name:       getName(p),
+						Properties: map[string]string{},
+						FullPath:   p,
+					},
+				})
+			}
+			continue
+		}
+
+		if strings.HasSuffix(obj.Key, directoryMarker) {
+			continue
+		}
+
+		if options.WithFiles && allowed {
+			attributes, err := c.bucket.Attributes(ctx, strings.ToLower(path))
+			if err != nil {
+				if gcerrors.Code(err) == gcerrors.NotFound {
+					attributes, err = c.bucket.Attributes(ctx, path)
+					if err != nil {
+						c.log.Error("Failed while retrieving attributes", "path", path, "err", err)
+						return nil, err
+					}
+				} else {
+					c.log.Error("Failed while retrieving attributes", "path", path, "err", err)
+					return nil, err
+				}
+			}
+
+			if attributes.ContentType == "application/x-directory; charset=UTF-8" {
+				// S3 directory representation
+				continue
+			}
+
+			if attributes.ContentType == "text/plain" && obj.Key == folderPath && attributes.Size == 0 {
+				// GCS directory representation
+				continue
+			}
+
+			var originalPath string
+			var props map[string]string
+			if attributes.Metadata != nil {
+				props = attributes.Metadata
+				if path, ok := attributes.Metadata[originalPathAttributeKey]; ok {
+					originalPath = path
+					delete(props, originalPathAttributeKey)
+				}
+			} else {
+				props = make(map[string]string)
+				originalPath = strings.TrimSuffix(path, Delimiter)
+			}
+
+			var contents []byte
+			if options.WithContents {
+				c, err := c.bucket.ReadAll(ctx, lowerPath)
+				if err != nil && gcerrors.Code(err) != gcerrors.NotFound {
+					return nil, err
+				}
+
+				if c != nil {
+					contents = c
+				}
+			}
+
+			files = append(files, &File{
+				Contents: contents,
+				FileMetadata: FileMetadata{
+					Name:       getName(originalPath),
+					FullPath:   originalPath,
+					Created:    attributes.CreateTime,
+					Properties: props,
+					Modified:   attributes.ModTime,
+					Size:       attributes.Size,
+					MimeType:   detectContentType(originalPath, attributes.ContentType),
+				},
+			})
+		}
+	}
+
+	hasMore := false
+	if len(files) > pageSize {
+		hasMore = true
+		files = files[:len(files)-pageSize]
+	}
+
+	lastPath := ""
+	if len(files) > 0 {
+		lastPath = files[len(files)-1].FullPath
+	}
+
+	return &ListResponse{
+		Files:    files,
+		HasMore:  hasMore,
+		LastPath: lastPath,
+	}, nil
+}
+
+func (c cdkBlobStorage) List(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListResponse, error) {
+	prefix := c.convertFolderPathToPrefix(folderPath)
+	return c.list(ctx, prefix, paging, options)
+}
+
 func (c cdkBlobStorage) close() error {
 	return c.bucket.Close()
 }

pkg/infra/filestorage/db_filestorage.go

@@ -2,7 +2,6 @@ package filestorage
 
 import (
 	"context"
-	"fmt"
 	"strings"
 	"time"
 
@@ -160,7 +159,7 @@ func (s dbFileStorage) Upsert(ctx context.Context, cmd *UpsertFileCommand) error
 
 			file := &file{
 				Path:             cmd.Path,
-				ParentFolderPath: getParentFolderPath(cmd.Path),
+				ParentFolderPath: strings.ToLower(getParentFolderPath(cmd.Path)),
 				Contents:         contentsToInsert,
 				MimeType:         cmd.MimeType,
 				Size:             int64(len(contentsToInsert)),
@@ -223,33 +222,85 @@ func upsertProperty(sess *sqlstore.DBSession, now time.Time, path string, key st
 	return err
 }
 
-func (s dbFileStorage) ListFiles(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListFilesResponse, error) {
-	var resp *ListFilesResponse
+//nolint: gocyclo
+func (s dbFileStorage) List(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListResponse, error) {
+	var resp *ListResponse
 
 	err := s.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
-		var foundFiles = make([]*file, 0)
-
-		sess.Table("file")
-		lowerFolderPath := strings.ToLower(folderPath)
-		if options.Recursive {
-			var nestedFolders string
-			if folderPath == Delimiter {
-				nestedFolders = "%"
+		cursor := ""
+		if paging != nil && paging.After != "" {
+			exists, err := sess.Table("file").Where("LOWER(path) = ?", strings.ToLower(paging.After)+Delimiter).Cols("mime_type").Exist()
+			if err != nil {
+				return err
+			}
+			if exists {
+				cursor = paging.After + Delimiter
 			} else {
-				nestedFolders = fmt.Sprintf("%s%s%s", lowerFolderPath, Delimiter, "%")
+				cursor = paging.After
 			}
-			sess.Where("(LOWER(parent_folder_path) = ?) OR (LOWER(parent_folder_path) LIKE ?)", lowerFolderPath, nestedFolders)
-		} else {
-			sess.Where("LOWER(parent_folder_path) = ?", lowerFolderPath)
 		}
-		sess.Where("LOWER(path) NOT LIKE ?", fmt.Sprintf("%s%s%s", "%", Delimiter, directoryMarker))
 
-		if options.PathFilters.isDenyAll() {
-			sess.Where("1 == 2")
+		var foundFiles = make([]*file, 0)
+		sess.Table("file")
+		lowerFolderPrefix := ""
+		lowerFolderPath := strings.ToLower(folderPath)
+		if lowerFolderPath == "" || lowerFolderPath == Delimiter {
+			lowerFolderPrefix = Delimiter
+			lowerFolderPath = Delimiter
 		} else {
+			lowerFolderPath = strings.TrimSuffix(lowerFolderPath, Delimiter)
+			lowerFolderPrefix = lowerFolderPath + Delimiter
+		}
+
+		sess.Where("LOWER(path) != ?", lowerFolderPrefix)
+
+		if !options.Recursive {
+			sess.Where("parent_folder_path = ?", lowerFolderPath)
+		} else {
+			sess.Where("(parent_folder_path = ?) OR (parent_folder_path LIKE ?)", lowerFolderPath, lowerFolderPrefix+"%")
+		}
+
+		if !options.WithFolders && options.WithFiles {
+			sess.Where("path NOT LIKE ?", "%/")
+		}
+
+		if options.WithFolders && !options.WithFiles {
+			sess.Where("path LIKE ?", "%/")
+		}
+
+		if len(options.PathFilters.allowedPrefixes)+len(options.PathFilters.allowedPaths) > 0 {
+			queries := make([]string, 0)
+			args := make([]interface{}, 0)
 			for _, prefix := range options.PathFilters.allowedPrefixes {
-				sess.Where("LOWER(path) LIKE ?", fmt.Sprintf("%s%s", strings.ToLower(prefix), "%"))
+				queries = append(queries, "LOWER(path) LIKE ?")
+				args = append(args, prefix+"%")
 			}
+
+			for _, path := range options.PathFilters.allowedPaths {
+				queries = append(queries, "LOWER(path) = ?")
+				args = append(args, path)
+			}
+			sess.Where(strings.Join(queries, " OR "), args...)
+		}
+
+		if options.PathFilters.disallowedPrefixes != nil && len(options.PathFilters.disallowedPrefixes) > 0 {
+			queries := make([]string, 0)
+			args := make([]interface{}, 0)
+			for _, prefix := range options.PathFilters.disallowedPrefixes {
+				queries = append(queries, "LOWER(path) NOT LIKE ?")
+				args = append(args, prefix+"%")
+			}
+			sess.Where(strings.Join(queries, " AND "), args...)
+		}
+
+		if options.PathFilters.disallowedPaths != nil && len(options.PathFilters.disallowedPaths) > 0 {
+			queries := make([]string, 0)
+			args := make([]interface{}, 0)
+			for _, path := range options.PathFilters.disallowedPaths {
+				queries = append(queries, "LOWER(path) != ?")
+				args = append(args, path)
+			}
+			sess.Where(strings.Join(queries, " AND "), args...)
 		}
 
 		sess.OrderBy("path")
@@ -257,8 +308,8 @@ func (s dbFileStorage) ListFiles(ctx context.Context, folderPath string, paging
 		pageSize := paging.First
 		sess.Limit(pageSize + 1)
 
-		if paging != nil && paging.After != "" {
-			sess.Where("path > ?", paging.After)
+		if cursor != "" {
+			sess.Where("path > ?", cursor)
 		}
 
 		if err := sess.Find(&foundFiles); err != nil {
@@ -272,24 +323,33 @@ func (s dbFileStorage) ListFiles(ctx context.Context, folderPath string, paging
 
 		lowerCasePaths := make([]string, 0)
 		for i := 0; i < foundLength; i++ {
-			lowerCasePaths = append(lowerCasePaths, strings.ToLower(foundFiles[i].Path))
+			isFolder := strings.HasSuffix(foundFiles[i].Path, Delimiter)
+			if !isFolder {
+				lowerCasePaths = append(lowerCasePaths, strings.ToLower(foundFiles[i].Path))
+			}
 		}
 		propertiesByLowerPath, err := s.getProperties(sess, lowerCasePaths)
 		if err != nil {
 			return err
 		}
 
-		files := make([]FileMetadata, 0)
+		files := make([]*File, 0)
 		for i := 0; i < foundLength; i++ {
 			var props map[string]string
-			path := foundFiles[i].Path
+			path := strings.TrimSuffix(foundFiles[i].Path, Delimiter)
 			if foundProps, ok := propertiesByLowerPath[strings.ToLower(path)]; ok {
 				props = foundProps
 			} else {
 				props = make(map[string]string)
 			}
 
-			files = append(files, FileMetadata{
+			var contents []byte
+			if options.WithContents {
+				contents = foundFiles[i].Contents
+			} else {
+				contents = []byte{}
+			}
+			files = append(files, &File{Contents: contents, FileMetadata: FileMetadata{
 				Name:       getName(path),
 				FullPath:   path,
 				Created:    foundFiles[i].Created,
@@ -297,7 +357,7 @@ func (s dbFileStorage) ListFiles(ctx context.Context, folderPath string, paging
 				Modified:   foundFiles[i].Updated,
 				Size:       foundFiles[i].Size,
 				MimeType:   foundFiles[i].MimeType,
-			})
+			}})
 		}
 
 		lastPath := ""
@@ -305,7 +365,7 @@ func (s dbFileStorage) ListFiles(ctx context.Context, folderPath string, paging
 			lastPath = files[len(files)-1].FullPath
 		}
 
-		resp = &ListFilesResponse{
+		resp = &ListResponse{
 			Files:    files,
 			LastPath: lastPath,
 			HasMore:  len(foundFiles) == pageSize+1,
@@ -316,67 +376,6 @@ func (s dbFileStorage) ListFiles(ctx context.Context, folderPath string, paging
 	return resp, err
 }
 
-func (s dbFileStorage) ListFolders(ctx context.Context, parentFolderPath string, options *ListOptions) ([]FileMetadata, error) {
-	folders := make([]FileMetadata, 0)
-
-	parentFolderPath = strings.TrimSuffix(parentFolderPath, Delimiter)
-	err := s.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
-		var foundPaths []string
-
-		sess.Table("file")
-		sess.Distinct("parent_folder_path")
-
-		if options.Recursive {
-			sess.Where("LOWER(parent_folder_path) > ?", strings.ToLower(parentFolderPath))
-		} else {
-			sess.Where("LOWER(parent_folder_path) LIKE ? AND LOWER(parent_folder_path) NOT LIKE ?", strings.ToLower(parentFolderPath)+Delimiter+"%", strings.ToLower(parentFolderPath)+Delimiter+"%"+Delimiter+"%")
-		}
-
-		if options.PathFilters.isDenyAll() {
-			sess.Where("1 == 2")
-		} else {
-			for _, prefix := range options.PathFilters.allowedPrefixes {
-				sess.Where("LOWER(parent_folder_path) LIKE ?", fmt.Sprintf("%s%s", strings.ToLower(prefix), "%"))
-			}
-		}
-
-		sess.OrderBy("parent_folder_path")
-		sess.Cols("parent_folder_path")
-
-		if err := sess.Find(&foundPaths); err != nil {
-			return err
-		}
-
-		mem := make(map[string]bool)
-		for i := 0; i < len(foundPaths); i++ {
-			path := foundPaths[i]
-			parts := strings.Split(path, Delimiter)
-			acc := parts[0]
-			j := 1
-			for {
-				acc = fmt.Sprintf("%s%s%s", acc, Delimiter, parts[j])
-				comparison := strings.Compare(acc, parentFolderPath)
-				if !mem[acc] && comparison > 0 {
-					folders = append(folders, FileMetadata{
-						Name:     getName(acc),
-						FullPath: acc,
-					})
-				}
-				mem[acc] = true
-
-				j += 1
-				if j >= len(parts) {
-					break
-				}
-			}
-		}
-
-		return nil
-	})
-
-	return folders, err
-}
-
 func (s dbFileStorage) CreateFolder(ctx context.Context, path string) error {
 	now := time.Now()
 	precedingFolders := precedingFolders(path)
@@ -384,29 +383,32 @@ func (s dbFileStorage) CreateFolder(ctx context.Context, path string) error {
 	err := s.db.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error {
 		var insertErr error
 		sess.MustLogSQL(true)
-		previousFolder := ""
+		previousFolder := Delimiter
 		for i := 0; i < len(precedingFolders); i++ {
 			existing := &file{}
-			directoryMarkerParentPath := previousFolder + Delimiter + getName(precedingFolders[i])
-			previousFolder = directoryMarkerParentPath
-			directoryMarkerPath := fmt.Sprintf("%s%s%s", directoryMarkerParentPath, Delimiter, directoryMarker)
-			lower := strings.ToLower(directoryMarkerPath)
-			exists, err := sess.Table("file").Where("LOWER(path) = ?", lower).Get(existing)
+			currentFolderParentPath := previousFolder
+			previousFolder = Join(previousFolder, getName(precedingFolders[i]))
+			currentFolderPath := previousFolder
+			if !strings.HasSuffix(currentFolderPath, Delimiter) {
+				currentFolderPath = currentFolderPath + Delimiter
+			}
+			exists, err := sess.Table("file").Where("LOWER(path) = ?", strings.ToLower(currentFolderPath)).Get(existing)
 			if err != nil {
 				insertErr = err
 				break
 			}
 
 			if exists {
-				previousFolder = existing.ParentFolderPath
+				previousFolder = strings.TrimSuffix(existing.Path, Delimiter)
 				continue
 			}
 
 			file := &file{
-				Path:             strings.ToLower(directoryMarkerPath),
-				ParentFolderPath: directoryMarkerParentPath,
+				Path:             currentFolderPath,
+				ParentFolderPath: strings.ToLower(currentFolderParentPath),
 				Contents:         make([]byte, 0),
 				Updated:          now,
+				MimeType:         DirectoryMimeType,
 				Created:          now,
 			}
 			_, err = sess.Insert(file)
@@ -433,8 +435,8 @@ func (s dbFileStorage) CreateFolder(ctx context.Context, path string) error {
 func (s dbFileStorage) DeleteFolder(ctx context.Context, folderPath string) error {
 	err := s.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
 		existing := &file{}
-		directoryMarkerPath := fmt.Sprintf("%s%s%s", folderPath, Delimiter, directoryMarker)
-		exists, err := sess.Table("file").Where("LOWER(path) = ?", strings.ToLower(directoryMarkerPath)).Get(existing)
+		internalFolderPath := strings.ToLower(folderPath) + Delimiter
+		exists, err := sess.Table("file").Where("LOWER(path) = ?", internalFolderPath).Get(existing)
 		if err != nil {
 			return err
 		}
@@ -443,7 +445,7 @@ func (s dbFileStorage) DeleteFolder(ctx context.Context, folderPath string) erro
 			return nil
 		}
 
-		_, err = sess.Table("file").Where("LOWER(path) = ?", strings.ToLower(directoryMarkerPath)).Delete(existing)
+		_, err = sess.Table("file").Where("LOWER(path) = ?", internalFolderPath).Delete(existing)
 		return err
 	})
 

pkg/infra/filestorage/fs_integration_test.go

@@ -70,6 +70,12 @@ func runTests(createCases func() []fsTestCase, t *testing.T) {
 	//	sqlStore = sqlstore.InitTestDB(t)
 	//	filestorage = NewDbStorage(testLogger, sqlStore, nil, "/")
 	//}
+	//
+	//setupSqlFSNestedPath := func() {
+	//	commonSetup()
+	//	sqlStore = sqlstore.InitTestDB(t)
+	//	filestorage = NewDbStorage(testLogger, sqlStore, nil, "/dashboards/")
+	//}
 
 	setupLocalFs := func() {
 		commonSetup()
@@ -127,6 +133,10 @@ func runTests(createCases func() []fsTestCase, t *testing.T) {
 		//	setup: setupSqlFS,
 		//	name:  "SQL FS",
 		//},
+		//{
+		//	setup: setupSqlFSNestedPath,
+		//	name:  "SQL FS with nested path",
+		//},
 	}
 
 	for _, backend := range backends {
@@ -180,11 +190,28 @@ func TestFsStorage(t *testing.T) {
 							checks(fPath("/folder1/folder2/file.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})),
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}},
+						list:  checks(listSize(4), listHasMore(false), listLastPath("/folder1/folder2/file.jpg")),
+						files: [][]interface{}{
+							checks(fPath("/folder1/file-inner.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})),
+							checks(fPath("/folder1/file-inner2.jpg"), fProperties(map[string]string{})),
+							checks(fPath("/folder1/folder2"), fProperties(map[string]string{}), fMimeType(DirectoryMimeType)),
+							checks(fPath("/folder1/folder2/file.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})),
+						},
+					},
 					queryListFiles{
 						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: false}},
 						list:  checks(listSize(0), listHasMore(false), listLastPath("")),
 						files: [][]interface{}{},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: false, WithFolders: true, WithFiles: true}},
+						list:  checks(listSize(1), listHasMore(false), listLastPath("/folder1")),
+						files: [][]interface{}{
+							checks(fPath("/folder1"), fProperties(map[string]string{}), fMimeType(DirectoryMimeType)),
+						},
+					},
 					queryListFiles{
 						input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: false}},
 						list:  checks(listSize(2), listHasMore(false), listLastPath("/folder1/file-inner2.jpg")),
@@ -193,6 +220,15 @@ func TestFsStorage(t *testing.T) {
 							checks(fPath("/folder1/file-inner2.jpg"), fProperties(map[string]string{})),
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: false, WithFolders: true, WithFiles: true}},
+						list:  checks(listSize(3), listHasMore(false), listLastPath("/folder1/folder2")),
+						files: [][]interface{}{
+							checks(fPath("/folder1/file-inner.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})),
+							checks(fPath("/folder1/file-inner2.jpg"), fProperties(map[string]string{})),
+							checks(fPath("/folder1/folder2"), fProperties(map[string]string{}), fMimeType(DirectoryMimeType)),
+						},
+					},
 					queryListFiles{
 						input: queryListFilesInput{path: "/folder1/folder2", options: &ListOptions{Recursive: false}},
 						list:  checks(listSize(1), listHasMore(false), listLastPath("/folder1/folder2/file.jpg")),
@@ -200,11 +236,23 @@ func TestFsStorage(t *testing.T) {
 							checks(fPath("/folder1/folder2/file.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})),
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/folder1/folder2", options: &ListOptions{Recursive: false, WithFolders: true, WithFiles: true}},
+						list:  checks(listSize(1), listHasMore(false), listLastPath("/folder1/folder2/file.jpg")),
+						files: [][]interface{}{
+							checks(fPath("/folder1/folder2/file.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})),
+						},
+					},
 					queryListFiles{
 						input: queryListFilesInput{path: "/folder1/folder2", options: &ListOptions{Recursive: false}, paging: &Paging{After: "/folder1/folder2/file.jpg"}},
 						list:  checks(listSize(0), listHasMore(false), listLastPath("")),
 						files: [][]interface{}{},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/folder1/folder2", options: &ListOptions{Recursive: false, WithFolders: true, WithFiles: true}, paging: &Paging{After: "/folder1/folder2/file.jpg"}},
+						list:  checks(listSize(0), listHasMore(false), listLastPath("")),
+						files: [][]interface{}{},
+					},
 				},
 			},
 			{
@@ -242,6 +290,15 @@ func TestFsStorage(t *testing.T) {
 							checks(fPath("/ab/a/a.jpg")),
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/ab", options: &ListOptions{Recursive: true, WithFolders: true, WithFiles: true}},
+						list:  checks(listSize(3), listHasMore(false), listLastPath("/ab/a/a.jpg")),
+						files: [][]interface{}{
+							checks(fPath("/ab/a.jpg")),
+							checks(fPath("/ab/a"), fMimeType(DirectoryMimeType)),
+							checks(fPath("/ab/a/a.jpg")),
+						},
+					},
 				},
 			},
 			{
@@ -265,10 +322,18 @@ func TestFsStorage(t *testing.T) {
 						input: queryListFilesInput{path: "/folder1/file-inner.jp", options: &ListOptions{Recursive: true}},
 						list:  checks(listSize(0), listHasMore(false), listLastPath("")),
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/folder1/file-inner.jp", options: &ListOptions{Recursive: true, WithFolders: true, WithFiles: true}},
+						list:  checks(listSize(0), listHasMore(false), listLastPath("")),
+					},
 					queryListFiles{
 						input: queryListFilesInput{path: "/folder1/file-inner", options: &ListOptions{Recursive: true}},
 						list:  checks(listSize(0), listHasMore(false), listLastPath("")),
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/folder1/file-inner", options: &ListOptions{Recursive: true, WithFolders: true, WithFiles: true}},
+						list:  checks(listSize(0), listHasMore(false), listLastPath("")),
+					},
 					queryListFiles{
 						input: queryListFilesInput{path: "/folder1/folder2/file.jpg", options: &ListOptions{Recursive: true}},
 						list:  checks(listSize(1), listHasMore(false), listLastPath("/folder1/folder2/file.jpg")),
@@ -276,6 +341,13 @@ func TestFsStorage(t *testing.T) {
 							checks(fPath("/folder1/folder2/file.jpg"), fName("file.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})),
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/folder1/folder2/file.jpg", options: &ListOptions{Recursive: true, WithFolders: true, WithFiles: true}},
+						list:  checks(listSize(1), listHasMore(false), listLastPath("/folder1/folder2/file.jpg")),
+						files: [][]interface{}{
+							checks(fPath("/folder1/folder2/file.jpg"), fName("file.jpg"), fProperties(map[string]string{"prop1": "val1", "prop2": "val"})),
+						},
+					},
 					queryListFiles{
 						input: queryListFilesInput{path: "/folder1/file-inner.jpg", options: &ListOptions{Recursive: true}},
 						list:  checks(listSize(1), listHasMore(false), listLastPath("/folder1/file-inner.jpg")),
@@ -283,6 +355,13 @@ func TestFsStorage(t *testing.T) {
 							checks(fPath("/folder1/file-inner.jpg"), fName("file-inner.jpg"), fProperties(map[string]string{"prop1": "val1"})),
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/folder1/file-inner.jpg", options: &ListOptions{Recursive: true, WithFolders: true, WithFiles: true}},
+						list:  checks(listSize(1), listHasMore(false), listLastPath("/folder1/file-inner.jpg")),
+						files: [][]interface{}{
+							checks(fPath("/folder1/file-inner.jpg"), fName("file-inner.jpg"), fProperties(map[string]string{"prop1": "val1"})),
+						},
+					},
 				},
 			},
 			{
@@ -304,6 +383,11 @@ func TestFsStorage(t *testing.T) {
 						input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, PathFilters: &PathFilters{allowedPrefixes: []string{"/folder2"}}}},
 						list:  checks(listSize(0), listHasMore(false), listLastPath("")),
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true, PathFilters: &PathFilters{allowedPrefixes: []string{"/folder2"}}}},
+						list:  checks(listSize(0), listHasMore(false), listLastPath("")),
+						files: [][]interface{}{},
+					},
 					queryListFiles{
 						input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, PathFilters: &PathFilters{allowedPrefixes: []string{"/folder1/folder"}}}},
 						list:  checks(listSize(1), listHasMore(false)),
@@ -311,6 +395,14 @@ func TestFsStorage(t *testing.T) {
 							checks(fPath("/folder1/folder2/file.jpg")),
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true, PathFilters: &PathFilters{allowedPrefixes: []string{"/folder1/folder"}}}},
+						list:  checks(listSize(2), listHasMore(false)),
+						files: [][]interface{}{
+							checks(fPath("/folder1/folder2"), fMimeType("directory")),
+							checks(fPath("/folder1/folder2/file.jpg")),
+						},
+					},
 				},
 			},
 			{
@@ -341,6 +433,20 @@ func TestFsStorage(t *testing.T) {
 							checks(fPath("/folder1/a")),
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: ""}},
+						list:  checks(listSize(1), listHasMore(true), listLastPath("/folder1")),
+						files: [][]interface{}{
+							checks(fPath("/folder1"), fMimeType(DirectoryMimeType)),
+						},
+					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: "/folder1"}},
+						list:  checks(listSize(1), listHasMore(true), listLastPath("/folder1/a")),
+						files: [][]interface{}{
+							checks(fPath("/folder1/a")),
+						},
+					},
 					queryListFiles{
 						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: "/folder1/a"}},
 						list:  checks(listSize(1), listHasMore(true), listLastPath("/folder1/b")),
@@ -348,6 +454,13 @@ func TestFsStorage(t *testing.T) {
 							checks(fPath("/folder1/b")),
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: "/folder1/a"}},
+						list:  checks(listSize(1), listHasMore(true), listLastPath("/folder1/b")),
+						files: [][]interface{}{
+							checks(fPath("/folder1/b")),
+						},
+					},
 					queryListFiles{
 						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: "/folder1/b"}},
 						list:  checks(listSize(1), listHasMore(false), listLastPath("/folder2/c")),
@@ -355,6 +468,20 @@ func TestFsStorage(t *testing.T) {
 							checks(fPath("/folder2/c")),
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: "/folder1/b"}},
+						list:  checks(listSize(1), listHasMore(true), listLastPath("/folder2")),
+						files: [][]interface{}{
+							checks(fPath("/folder2"), fMimeType(DirectoryMimeType)),
+						},
+					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: "/folder2"}},
+						list:  checks(listSize(1), listHasMore(false), listLastPath("/folder2/c")),
+						files: [][]interface{}{
+							checks(fPath("/folder2/c")),
+						},
+					},
 					queryListFiles{
 						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: ""}},
 						list:  checks(listSize(3), listHasMore(false), listLastPath("/folder2/c")),
@@ -364,14 +491,33 @@ func TestFsStorage(t *testing.T) {
 							checks(fPath("/folder2/c")),
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 5, After: ""}},
+						list:  checks(listSize(5), listHasMore(false), listLastPath("/folder2/c")),
+						files: [][]interface{}{
+							checks(fPath("/folder1"), fMimeType(DirectoryMimeType)),
+							checks(fPath("/folder1/a")),
+							checks(fPath("/folder1/b")),
+							checks(fPath("/folder2"), fMimeType(DirectoryMimeType)),
+							checks(fPath("/folder2/c")),
+						},
+					},
 					queryListFiles{
 						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: "/folder2"}},
 						list:  checks(listSize(1), listHasMore(false)),
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 5, After: "/folder2"}},
+						list:  checks(listSize(1), listHasMore(false)),
+					},
 					queryListFiles{
 						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: "/folder2/c"}},
 						list:  checks(listSize(0), listHasMore(false)),
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 5, After: "/folder2/c"}},
+						list:  checks(listSize(0), listHasMore(false)),
+					},
 				},
 			},
 		}
@@ -417,6 +563,18 @@ func TestFsStorage(t *testing.T) {
 							checks(fPath("/folderX/folderZ")),
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: false, WithFolders: true}},
+						list:  checks(listSize(6), listHasMore(false), listLastPath("/folderX/folderZ")),
+						files: [][]interface{}{
+							checks(fPath("/folder1"), fMimeType(DirectoryMimeType)),
+							checks(fPath("/folder1/folder2"), fMimeType(DirectoryMimeType)),
+							checks(fPath("/folderA"), fMimeType(DirectoryMimeType)),
+							checks(fPath("/folderA/folderB"), fMimeType(DirectoryMimeType)),
+							checks(fPath("/folderX"), fMimeType(DirectoryMimeType)),
+							checks(fPath("/folderX/folderZ"), fMimeType(DirectoryMimeType)),
+						},
+					},
 				},
 			},
 			{
@@ -452,10 +610,27 @@ func TestFsStorage(t *testing.T) {
 							checks(fPath("/folder1/folder2")),
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: false, WithFiles: false, WithFolders: true}},
+						list:  checks(listSize(1), listHasMore(false), listLastPath("/folder1/folder2")),
+						files: [][]interface{}{
+							checks(fPath("/folder1/folder2"), fMimeType(DirectoryMimeType)),
+						},
+					},
 					queryListFolders{
 						input:  queryListFoldersInput{path: "/folderZ", options: &ListOptions{Recursive: false}},
 						checks: [][]interface{}{},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/folderZ", options: &ListOptions{Recursive: false, WithFiles: false, WithFolders: true}},
+						list:  checks(listSize(0), listHasMore(false), listLastPath("")),
+						files: [][]interface{}{},
+					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/folder1/folder2", options: &ListOptions{Recursive: false, WithFiles: false, WithFolders: true}},
+						list:  checks(listSize(0), listHasMore(false), listLastPath("")),
+						files: [][]interface{}{},
+					},
 					queryListFolders{
 						input: queryListFoldersInput{path: "/", options: &ListOptions{Recursive: false}},
 						checks: [][]interface{}{
@@ -464,6 +639,15 @@ func TestFsStorage(t *testing.T) {
 							checks(fPath("/folderX")),
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: false, WithFiles: false, WithFolders: true}},
+						list:  checks(listSize(3), listHasMore(false), listLastPath("/folderX")),
+						files: [][]interface{}{
+							checks(fPath("/folder1"), fMimeType(DirectoryMimeType)),
+							checks(fPath("/folderA"), fMimeType(DirectoryMimeType)),
+							checks(fPath("/folderX"), fMimeType(DirectoryMimeType)),
+						},
+					},
 				},
 			},
 		}
@@ -1047,6 +1231,31 @@ func TestFsStorage(t *testing.T) {
 							// /s3/folder/nested/dashboard.json is denied with '/s3/folder/nested/' prefix
 						},
 					},
+					queryListFiles{
+						input: queryListFilesInput{path: "/", options: &ListOptions{
+							Recursive:   true,
+							PathFilters: pathFilters,
+							WithFiles:   true,
+							WithFolders: true,
+						}},
+						list: checks(listSize(10), listHasMore(false), listLastPath("/s3/folder/dashboard.json")),
+						files: [][]interface{}{
+							// /gitA/dashboard.json is not explicitly allowed
+							checks(fPath("/gitA/dashboard2.json")),         // explicitly allowed by allowedPath
+							checks(fPath("/gitB")),                         // allowed by '/gitB/' prefix
+							checks(fPath("/gitB/nested")),                  // allowed by '/gitB/' prefix
+							checks(fPath("/gitB/nested/dashboard.json")),   // allowed by '/gitB/' prefix
+							checks(fPath("/gitB/nested2")),                 // allowed by '/gitB/' prefix
+							checks(fPath("/gitB/nested2/dashboard2.json")), // allowed by '/gitB/' prefix
+							checks(fPath("/gitC")),                         // allowed by '/gitC/' prefix
+							// /gitC/nestedC is explicitly denied
+							checks(fPath("/gitC/nestedC/dashboardC.json")), // allowed by '/gitC/' prefix
+							// /s3 is not explicitly allowed
+							checks(fPath("/s3/folder")),
+							checks(fPath("/s3/folder/dashboard.json")), // allowed by '/s3/folder/' prefix
+							// /s3/folder/nested/dashboard.json is denied with '/s3/folder/nested/' prefix
+						},
+					},
 					queryListFolders{
 						input: queryListFoldersInput{path: "/", options: &ListOptions{
 							Recursive:   true,

pkg/infra/filestorage/test_utils.go

@@ -137,6 +137,7 @@ type queryListFiles struct {
 
 type queryListFoldersInput struct {
 	path    string
+	paging  *Paging
 	options *ListOptions
 }
 
@@ -221,7 +222,7 @@ func runChecks(t *testing.T, stepName string, path string, output interface{}, c
 	}
 
 	switch o := output.(type) {
-	case File:
+	case *File:
 		for _, check := range checks {
 			checkName := interfaceName(check)
 			if fileContentsCheck, ok := check.(fileContentsCheck); ok {
@@ -234,7 +235,7 @@ func runChecks(t *testing.T, stepName string, path string, output interface{}, c
 		for _, check := range checks {
 			runFileMetadataCheck(o, check, interfaceName(check))
 		}
-	case ListFilesResponse:
+	case ListResponse:
 		for _, check := range checks {
 			c := check
 			checkName := interfaceName(c)
@@ -249,13 +250,14 @@ func runChecks(t *testing.T, stepName string, path string, output interface{}, c
 				t.Fatalf("unrecognized list check %s", checkName)
 			}
 		}
+
 	default:
 		t.Fatalf("unrecognized output %s", interfaceName(output))
 	}
 
 }
 
-func formatPathStructure(files []FileMetadata) string {
+func formatPathStructure(files []*File) string {
 	if len(files) == 0 {
 		return "<<EMPTY>>"
 	}
@@ -278,13 +280,13 @@ func handleQuery(t *testing.T, ctx context.Context, query interface{}, queryName
 		if q.checks != nil && len(q.checks) > 0 {
 			require.NotNil(t, file, "%s %s", queryName, inputPath)
 			require.Equal(t, strings.ToLower(inputPath), strings.ToLower(file.FullPath), "%s %s", queryName, inputPath)
-			runChecks(t, queryName, inputPath, *file, q.checks)
+			runChecks(t, queryName, inputPath, file, q.checks)
 		} else {
 			require.Nil(t, file, "%s %s", queryName, inputPath)
 		}
 	case queryListFiles:
 		inputPath := q.input.path
-		resp, err := fs.ListFiles(ctx, inputPath, q.input.paging, q.input.options)
+		resp, err := fs.List(ctx, inputPath, q.input.paging, q.input.options)
 		require.NoError(t, err, "%s: should be able to list files in %s", queryName, inputPath)
 		require.NotNil(t, resp)
 		if q.list != nil && len(q.list) > 0 {
@@ -304,17 +306,33 @@ func handleQuery(t *testing.T, ctx context.Context, query interface{}, queryName
 		}
 	case queryListFolders:
 		inputPath := q.input.path
-		resp, err := fs.ListFolders(ctx, inputPath, q.input.options)
+		opts := q.input.options
+		if opts == nil {
+			opts = &ListOptions{
+				Recursive:    true,
+				WithFiles:    false,
+				WithFolders:  true,
+				WithContents: false,
+				PathFilters:  nil,
+			}
+		} else {
+			opts.WithFolders = true
+			opts.WithFiles = false
+		}
+		resp, err := fs.List(ctx, inputPath, &Paging{
+			After: "",
+			First: 100000,
+		}, opts)
 		require.NotNil(t, resp)
 		require.NoError(t, err, "%s: should be able to list folders in %s", queryName, inputPath)
 
 		if q.checks != nil {
-			require.Equal(t, len(resp), len(q.checks), "%s: expected a check for each actual folder at path: \"%s\". actual: %s", queryName, inputPath, formatPathStructure(resp))
-			for i, file := range resp {
+			require.Equal(t, len(resp.Files), len(q.checks), "%s: expected a check for each actual folder at path: \"%s\". actual: %s", queryName, inputPath, formatPathStructure(resp.Files))
+			for i, file := range resp.Files {
 				runChecks(t, queryName, inputPath, file, q.checks[i])
 			}
 		} else {
-			require.Equal(t, 0, len(resp), "%s %s", queryName, inputPath)
+			require.Equal(t, 0, len(resp.Files), "%s %s", queryName, inputPath)
 		}
 	default:
 		t.Fatalf("unrecognized query %s", queryName)

pkg/infra/filestorage/wrapper.go

@@ -202,6 +202,10 @@ func (b wrapper) Get(ctx context.Context, path string) (*File, error) {
 		return nil, nil
 	}
 
+	if b.rootFolder == rootedPath {
+		return nil, nil
+	}
+
 	file, err := b.wrapped.Get(ctx, rootedPath)
 	if file != nil {
 		file.FullPath = b.removeRoot(file.FullPath)
@@ -276,90 +280,41 @@ func (b wrapper) pagingOptionsWithDefaults(paging *Paging) *Paging {
 	return paging
 }
 
-func (b wrapper) listOptionsWithDefaults(options *ListOptions, folderQuery bool) *ListOptions {
+func (b wrapper) listOptionsWithDefaults(options *ListOptions) *ListOptions {
 	if options == nil {
-		options = &ListOptions{}
-		options.Recursive = folderQuery
-		options.PathFilters = b.pathFilters
-
 		return &ListOptions{
-			Recursive:   folderQuery,
-			PathFilters: b.pathFilters,
+			Recursive:    false,
+			PathFilters:  b.pathFilters,
+			WithFiles:    true,
+			WithFolders:  false,
+			WithContents: false,
 		}
 	}
 
+	withFiles := options.WithFiles
+	if !options.WithFiles && !options.WithFolders {
+		withFiles = true
+	}
 	if options.PathFilters == nil {
 		return &ListOptions{
-			Recursive:   options.Recursive,
-			PathFilters: b.pathFilters,
+			Recursive:    options.Recursive,
+			PathFilters:  b.pathFilters,
+			WithFiles:    withFiles,
+			WithFolders:  options.WithFolders,
+			WithContents: options.WithContents,
 		}
 	}
 
 	rootedFilters := addRootFolderToFilters(copyPathFilters(options.PathFilters), b.rootFolder)
 	return &ListOptions{
-		Recursive:   options.Recursive,
-		PathFilters: addPathFilters(rootedFilters, b.pathFilters),
+		Recursive:    options.Recursive,
+		PathFilters:  addPathFilters(rootedFilters, b.pathFilters),
+		WithFiles:    withFiles,
+		WithFolders:  options.WithFolders,
+		WithContents: options.WithContents,
 	}
 }
 
-func (b wrapper) ListFiles(ctx context.Context, path string, paging *Paging, options *ListOptions) (*ListFilesResponse, error) {
-	if err := b.validatePath(path); err != nil {
-		return nil, err
-	}
-
-	pathWithRoot := b.addRoot(path)
-	resp, err := b.wrapped.ListFiles(ctx, pathWithRoot, b.pagingOptionsWithDefaults(paging), b.listOptionsWithDefaults(options, false))
-
-	if resp != nil && resp.Files != nil {
-		if resp.LastPath != "" {
-			resp.LastPath = b.removeRoot(resp.LastPath)
-		}
-
-		for i := 0; i < len(resp.Files); i++ {
-			resp.Files[i].FullPath = b.removeRoot(resp.Files[i].FullPath)
-		}
-	}
-
-	if err != nil {
-		return resp, err
-	}
-
-	if len(resp.Files) != 0 {
-		return resp, err
-	}
-
-	// TODO: optimize, don't fetch the contents in this case
-	file, err := b.Get(ctx, path)
-	if err != nil {
-		return resp, err
-	}
-
-	if file != nil {
-		file.FileMetadata.FullPath = b.removeRoot(file.FileMetadata.FullPath)
-		return &ListFilesResponse{
-			Files:    []FileMetadata{file.FileMetadata},
-			HasMore:  false,
-			LastPath: file.FileMetadata.FullPath,
-		}, nil
-	}
-
-	return resp, err
-}
-
-func (b wrapper) ListFolders(ctx context.Context, path string, options *ListOptions) ([]FileMetadata, error) {
-	if err := b.validatePath(path); err != nil {
-		return nil, err
-	}
-
-	folders, err := b.wrapped.ListFolders(ctx, b.addRoot(path), b.listOptionsWithDefaults(options, true))
-	if folders != nil {
-		for i := 0; i < len(folders); i++ {
-			folders[i].FullPath = b.removeRoot(folders[i].FullPath)
-		}
-	}
-	return folders, err
-}
-
 func (b wrapper) CreateFolder(ctx context.Context, path string) error {
 	if err := b.validatePath(path); err != nil {
 		return err
@@ -395,24 +350,77 @@ func (b wrapper) DeleteFolder(ctx context.Context, path string) error {
 	return b.wrapped.DeleteFolder(ctx, rootedPath)
 }
 
+func (b wrapper) List(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListResponse, error) {
+	if err := b.validatePath(folderPath); err != nil {
+		return nil, err
+	}
+
+	options = b.listOptionsWithDefaults(options)
+	if (!options.WithFiles && !options.WithFolders) || options.isDenyAll() {
+		return &ListResponse{
+			Files:    []*File{},
+			HasMore:  false,
+			LastPath: "",
+		}, nil
+	}
+
+	var fileChan = make(chan *File)
+	fileRetrievalCtx, cancelFileGet := context.WithCancel(ctx)
+	defer cancelFileGet()
+
+	go func() {
+		if options.WithFiles {
+			if f, err := b.Get(fileRetrievalCtx, folderPath); err == nil {
+				fileChan <- f
+				return
+			}
+		}
+		fileChan <- nil
+	}()
+
+	pathWithRoot := b.addRoot(folderPath)
+	resp, err := b.wrapped.List(ctx, pathWithRoot, b.pagingOptionsWithDefaults(paging), options)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp != nil && resp.Files != nil && len(resp.Files) > 0 {
+		if resp.LastPath != "" {
+			resp.LastPath = b.removeRoot(resp.LastPath)
+		}
+
+		for i := 0; i < len(resp.Files); i++ {
+			resp.Files[i].FullPath = b.removeRoot(resp.Files[i].FullPath)
+		}
+		return resp, err
+	}
+
+	file := <-fileChan
+	if file != nil {
+		file.FileMetadata.FullPath = b.removeRoot(file.FileMetadata.FullPath)
+		var contents []byte
+		if options.WithContents {
+			contents = file.Contents
+		} else {
+			contents = []byte{}
+		}
+		return &ListResponse{
+			Files:    []*File{{Contents: contents, FileMetadata: file.FileMetadata}},
+			HasMore:  false,
+			LastPath: file.FileMetadata.FullPath,
+		}, nil
+	}
+
+	return resp, err
+}
+
 func (b wrapper) isFolderEmpty(ctx context.Context, path string) (bool, error) {
-	filesInFolder, err := b.ListFiles(ctx, path, &Paging{First: 1}, &ListOptions{Recursive: true})
+	resp, err := b.List(ctx, path, &Paging{First: 1}, &ListOptions{Recursive: true, WithFolders: true, WithFiles: true})
 	if err != nil {
 		return false, err
 	}
 
-	if len(filesInFolder.Files) > 0 {
-		return false, nil
-	}
-
-	folders, err := b.ListFolders(ctx, path, &ListOptions{
-		Recursive: true,
-	})
-	if err != nil {
-		return false, err
-	}
-
-	if len(folders) > 0 {
+	if len(resp.Files) > 0 {
 		return false, nil
 	}