Storage: add delete / deleteFolder / createFolder (#51887)

* delete / delete folder / create folder * add backend tests * implement force delete * fix merge * lint fix * fix delete root folder * fix folder name validation * fix mysql path_hash issue * Fix returning error
2025-02-25 18:55:37 -06:00 · 2022-07-08 22:23:16 +04:00
parent e51187a474
commit 1d2aa7c69b
14 changed files with 853 additions and 103 deletions
--- a/pkg/services/store/http.go
+++ b/pkg/services/store/http.go
@@ -1,8 +1,10 @@
 package store

 import (
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"strings"
@@ -20,6 +22,8 @@ type HTTPStorageService interface {
 	List(c *models.ReqContext) response.Response
 	Read(c *models.ReqContext) response.Response
 	Delete(c *models.ReqContext) response.Response
+	DeleteFolder(c *models.ReqContext) response.Response
+	CreateFolder(c *models.ReqContext) response.Response
 	Upload(c *models.ReqContext) response.Response
 }

@@ -71,6 +75,14 @@ func (s *httpStorage) Upload(c *models.ReqContext) response.Response {
 		})
 	}

+	folder, ok := c.Req.MultipartForm.Value["folder"]
+	if !ok || len(folder) != 1 {
+		return response.JSON(400, map[string]interface{}{
+			"message": "please specify the upload folder",
+			"err":     true,
+		})
+	}
+
 	fileHeader := files[0]
 	if fileHeader.Size > MAX_UPLOAD_SIZE {
 		return errFileTooBig
@@ -95,7 +107,7 @@ func (s *httpStorage) Upload(c *models.ReqContext) response.Response {
 		return errFileTooBig
 	}

-	path := RootResources + "/" + fileHeader.Filename
+	path := folder[0] + "/" + fileHeader.Filename

 	mimeType := http.DetectContentType(data)

@@ -140,17 +152,75 @@ func (s *httpStorage) Read(c *models.ReqContext) response.Response {

 func (s *httpStorage) Delete(c *models.ReqContext) response.Response {
 	// full path is api/storage/delete/upload/example.jpg, but we only want the part after upload
-	_, path := getPathAndScope(c)
-	err := s.store.Delete(c.Req.Context(), c.SignedInUser, "/"+path)
+	scope, path := getPathAndScope(c)
+
+	err := s.store.Delete(c.Req.Context(), c.SignedInUser, scope+"/"+path)
 	if err != nil {
-		return response.Error(400, "cannot call delete", err)
+		return response.Error(400, "failed to delete the file: "+err.Error(), err)
 	}
-	return response.JSON(200, map[string]string{
+	return response.JSON(200, map[string]interface{}{
 		"message": "Removed file from storage",
+		"success": true,
 		"path":    path,
 	})
 }

+func (s *httpStorage) DeleteFolder(c *models.ReqContext) response.Response {
+	body, err := io.ReadAll(c.Req.Body)
+	if err != nil {
+		return response.Error(500, "error reading bytes", err)
+	}
+
+	cmd := &DeleteFolderCmd{}
+	err = json.Unmarshal(body, cmd)
+	if err != nil {
+		return response.Error(400, "error parsing body", err)
+	}
+
+	if cmd.Path == "" {
+		return response.Error(400, "empty path", err)
+	}
+
+	// full path is api/storage/delete/upload/example.jpg, but we only want the part after upload
+	_, path := getPathAndScope(c)
+	if err := s.store.DeleteFolder(c.Req.Context(), c.SignedInUser, cmd); err != nil {
+		return response.Error(400, "failed to delete the folder: "+err.Error(), err)
+	}
+
+	return response.JSON(200, map[string]interface{}{
+		"message": "Removed folder from storage",
+		"success": true,
+		"path":    path,
+	})
+}
+
+func (s *httpStorage) CreateFolder(c *models.ReqContext) response.Response {
+	body, err := io.ReadAll(c.Req.Body)
+	if err != nil {
+		return response.Error(500, "error reading bytes", err)
+	}
+
+	cmd := &CreateFolderCmd{}
+	err = json.Unmarshal(body, cmd)
+	if err != nil {
+		return response.Error(400, "error parsing body", err)
+	}
+
+	if cmd.Path == "" {
+		return response.Error(400, "empty path", err)
+	}
+
+	if err := s.store.CreateFolder(c.Req.Context(), c.SignedInUser, cmd); err != nil {
+		return response.Error(400, "failed to create the folder: "+err.Error(), err)
+	}
+
+	return response.JSON(200, map[string]interface{}{
+		"message": "Folder created",
+		"success": true,
+		"path":    cmd.Path,
+	})
+}
+
 func (s *httpStorage) List(c *models.ReqContext) response.Response {
 	params := web.Params(c.Req)
 	path := params["*"]
--- a/pkg/services/store/service.go
+++ b/pkg/services/store/service.go
@@ -19,7 +19,7 @@ import (
 var grafanaStorageLogger = log.New("grafanaStorageLogger")

 var ErrUploadFeatureDisabled = errors.New("upload feature is disabled")
-var ErrUnsupportedStorage = errors.New("storage does not support upload operation")
+var ErrUnsupportedStorage = errors.New("storage does not support this operation")
 var ErrUploadInternalError = errors.New("upload internal error")
 var ErrValidationFailed = errors.New("request validation failed")
 var ErrFileAlreadyExists = errors.New("file exists")
@@ -29,6 +29,15 @@ const RootResources = "resources"

 const MAX_UPLOAD_SIZE = 3 * 1024 * 1024 // 3MB

+type DeleteFolderCmd struct {
+	Path  string `json:"path"`
+	Force bool   `json:"force"`
+}
+
+type CreateFolderCmd struct {
+	Path string `json:"path"`
+}
+
 type StorageService interface {
 	registry.BackgroundService

@@ -42,6 +51,10 @@ type StorageService interface {

 	Delete(ctx context.Context, user *models.SignedInUser, path string) error

+	DeleteFolder(ctx context.Context, user *models.SignedInUser, cmd *DeleteFolderCmd) error
+
+	CreateFolder(ctx context.Context, user *models.SignedInUser, cmd *CreateFolderCmd) error
+
 	validateUploadRequest(ctx context.Context, user *models.SignedInUser, req *UploadRequest, storagePath string) validationResult

 	// sanitizeUploadRequest sanitizes the upload request and converts it into a command accepted by the FileStorage API
@@ -86,15 +99,10 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles,
 		return storages
 	}

-	s := newStandardStorageService(globalRoots, initializeOrgStorages)
-	s.sql = sql
-	s.cfg = storageServiceConfig{
-		allowUnsanitizedSvgUpload: false,
-	}
-	return s
+	return newStandardStorageService(sql, globalRoots, initializeOrgStorages)
 }

-func newStandardStorageService(globalRoots []storageRuntime, initializeOrgStorages func(orgId int64) []storageRuntime) *standardStorageService {
+func newStandardStorageService(sql *sqlstore.SQLStore, globalRoots []storageRuntime, initializeOrgStorages func(orgId int64) []storageRuntime) *standardStorageService {
 	rootsByOrgId := make(map[int64][]storageRuntime)
 	rootsByOrgId[ac.GlobalOrgID] = globalRoots

@@ -104,7 +112,11 @@ func newStandardStorageService(globalRoots []storageRuntime, initializeOrgStorag
 	}
 	res.init()
 	return &standardStorageService{
+		sql:  sql,
 		tree: res,
+		cfg: storageServiceConfig{
+			allowUnsanitizedSvgUpload: false,
+		},
 	}
 }

@@ -143,13 +155,18 @@ type UploadRequest struct {
 	OverwriteExistingFile bool
 }

+func storageSupportsMutatingOperations(path string) bool {
+	// TODO: this is temporary - make it rbac-driven
+	return strings.HasPrefix(path, RootResources+"/") || path == RootResources
+}
+
 func (s *standardStorageService) Upload(ctx context.Context, user *models.SignedInUser, req *UploadRequest) error {
 	upload, _ := s.tree.getRoot(getOrgId(user), RootResources)
 	if upload == nil {
 		return ErrUploadFeatureDisabled
 	}

-	if !strings.HasPrefix(req.Path, RootResources+"/") {
+	if !storageSupportsMutatingOperations(req.Path) {
 		return ErrUnsupportedStorage
 	}

@@ -188,12 +205,53 @@ func (s *standardStorageService) Upload(ctx context.Context, user *models.Signed
 	return nil
 }

-func (s *standardStorageService) Delete(ctx context.Context, user *models.SignedInUser, path string) error {
-	upload, _ := s.tree.getRoot(getOrgId(user), RootResources)
-	if upload == nil {
-		return fmt.Errorf("upload feature is not enabled")
+func (s *standardStorageService) DeleteFolder(ctx context.Context, user *models.SignedInUser, cmd *DeleteFolderCmd) error {
+	resources, _ := s.tree.getRoot(getOrgId(user), RootResources)
+	if resources == nil {
+		return fmt.Errorf("resources storage is not enabled")
 	}
-	err := upload.Delete(ctx, path)
+
+	if !storageSupportsMutatingOperations(cmd.Path) {
+		return ErrUnsupportedStorage
+	}
+
+	storagePath := strings.TrimPrefix(cmd.Path, RootResources)
+	if storagePath == "" {
+		storagePath = filestorage.Delimiter
+	}
+	return resources.DeleteFolder(ctx, storagePath, &filestorage.DeleteFolderOptions{Force: true})
+}
+
+func (s *standardStorageService) CreateFolder(ctx context.Context, user *models.SignedInUser, cmd *CreateFolderCmd) error {
+	if !storageSupportsMutatingOperations(cmd.Path) {
+		return ErrUnsupportedStorage
+	}
+
+	resources, _ := s.tree.getRoot(getOrgId(user), RootResources)
+	if resources == nil {
+		return fmt.Errorf("resources storage is not enabled")
+	}
+
+	storagePath := strings.TrimPrefix(cmd.Path, RootResources)
+	err := resources.CreateFolder(ctx, storagePath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *standardStorageService) Delete(ctx context.Context, user *models.SignedInUser, path string) error {
+	if !storageSupportsMutatingOperations(path) {
+		return ErrUnsupportedStorage
+	}
+
+	resources, _ := s.tree.getRoot(getOrgId(user), RootResources)
+	if resources == nil {
+		return fmt.Errorf("resources storage is not enabled")
+	}
+
+	storagePath := strings.TrimPrefix(path, RootResources)
+	err := resources.Delete(ctx, storagePath)
 	if err != nil {
 		return err
 	}
--- a/pkg/services/store/service_test.go
+++ b/pkg/services/store/service_test.go
@@ -3,16 +3,15 @@ package store
 import (
 	"bytes"
 	"context"
-	"os"
 	"path/filepath"
 	"testing"

 	"github.com/grafana/grafana-plugin-sdk-go/experimental"
+	"github.com/grafana/grafana/pkg/infra/filestorage"
 	"github.com/grafana/grafana/pkg/models"
-	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
-	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb/testdatasource"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )

@@ -37,7 +36,7 @@ func TestListFiles(t *testing.T) {
 		}).setReadOnly(true).setBuiltin(true),
 	}

-	store := newStandardStorageService(roots, func(orgId int64) []storageRuntime {
+	store := newStandardStorageService(sqlstore.InitTestDB(t), roots, func(orgId int64) []storageRuntime {
 		return make([]storageRuntime, 0)
 	})
 	frame, err := store.List(context.Background(), dummyUser, "public/testdata")
@@ -54,18 +53,75 @@ func TestListFiles(t *testing.T) {
 	experimental.CheckGoldenJSONFrame(t, "testdata", "public_testdata_js_libraries.golden", testDsFrame, true)
 }

-func TestUpload(t *testing.T) {
-	features := featuremgmt.WithFeatures(featuremgmt.FlagStorageLocalUpload)
-	path, err := os.Getwd()
-	require.NoError(t, err)
-	cfg := &setting.Cfg{AppURL: "http://localhost:3000/", DataPath: path}
-	s := ProvideService(sqlstore.InitTestDB(t), features, cfg)
-	request := UploadRequest{
+func setupUploadStore(t *testing.T) (StorageService, *filestorage.MockFileStorage, string) {
+	t.Helper()
+	storageName := "resources"
+	mockStorage := &filestorage.MockFileStorage{}
+	sqlStorage := newSQLStorage(storageName, "Testing upload", &StorageSQLConfig{orgId: 1}, sqlstore.InitTestDB(t))
+	sqlStorage.store = mockStorage
+
+	store := newStandardStorageService(sqlstore.InitTestDB(t), []storageRuntime{sqlStorage}, func(orgId int64) []storageRuntime {
+		return make([]storageRuntime, 0)
+	})
+
+	return store, mockStorage, storageName
+}
+
+func TestShouldUploadWhenNoFileAlreadyExists(t *testing.T) {
+	service, mockStorage, storageName := setupUploadStore(t)
+
+	mockStorage.On("Get", mock.Anything, "/myFile.jpg").Return(nil, nil)
+	mockStorage.On("Upsert", mock.Anything, mock.Anything).Return(nil)
+
+	err := service.Upload(context.Background(), dummyUser, &UploadRequest{
 		EntityType: EntityTypeImage,
 		Contents:   make([]byte, 0),
-		Path:       "resources/myFile.jpg",
+		Path:       storageName + "/myFile.jpg",
 		MimeType:   "image/jpg",
-	}
-	err = s.Upload(context.Background(), dummyUser, &request)
+	})
+	require.NoError(t, err)
+}
+
+func TestShouldFailUploadWhenFileAlreadyExists(t *testing.T) {
+	service, mockStorage, storageName := setupUploadStore(t)
+
+	mockStorage.On("Get", mock.Anything, "/myFile.jpg").Return(&filestorage.File{Contents: make([]byte, 0)}, nil)
+
+	err := service.Upload(context.Background(), dummyUser, &UploadRequest{
+		EntityType: EntityTypeImage,
+		Contents:   make([]byte, 0),
+		Path:       storageName + "/myFile.jpg",
+		MimeType:   "image/jpg",
+	})
+	require.ErrorIs(t, err, ErrFileAlreadyExists)
+}
+
+func TestShouldDelegateFileDeletion(t *testing.T) {
+	service, mockStorage, storageName := setupUploadStore(t)
+
+	mockStorage.On("Delete", mock.Anything, "/myFile.jpg").Return(nil)
+
+	err := service.Delete(context.Background(), dummyUser, storageName+"/myFile.jpg")
+	require.NoError(t, err)
+}
+
+func TestShouldDelegateFolderCreation(t *testing.T) {
+	service, mockStorage, storageName := setupUploadStore(t)
+
+	mockStorage.On("CreateFolder", mock.Anything, "/nestedFolder/mostNestedFolder").Return(nil)
+
+	err := service.CreateFolder(context.Background(), dummyUser, &CreateFolderCmd{Path: storageName + "/nestedFolder/mostNestedFolder"})
+	require.NoError(t, err)
+}
+
+func TestShouldDelegateFolderDeletion(t *testing.T) {
+	service, mockStorage, storageName := setupUploadStore(t)
+
+	mockStorage.On("DeleteFolder", mock.Anything, "/", &filestorage.DeleteFolderOptions{Force: true}).Return(nil)
+
+	err := service.DeleteFolder(context.Background(), dummyUser, &DeleteFolderCmd{
+		Path:  storageName,
+		Force: true,
+	})
 	require.NoError(t, err)
 }