Storage: refactor readonly support (#52127)

pkg/services/store/entity_events.go

@@ -26,6 +26,7 @@ const (
 	EntityTypeDashboard EntityType = "dashboard"
 	EntityTypeFolder    EntityType = "folder"
 	EntityTypeImage     EntityType = "image"
+	EntityTypeJSON      EntityType = "json"
 )
 
 // CreateDatabaseEntityId creates entityId for entities stored in the existing SQL tables

pkg/services/store/http.go

@@ -15,8 +15,6 @@ import (
 	"github.com/grafana/grafana/pkg/web"
 )
 
-var errFileTooBig = response.Error(400, "Please limit file uploaded under 1MB", errors.New("file is too big"))
-
 // HTTPStorageService passes raw HTTP requests to a well typed storage service
 type HTTPStorageService interface {
 	List(c *models.ReqContext) response.Response
@@ -57,38 +55,37 @@ func UploadErrorToStatusCode(err error) int {
 }
 
 func (s *httpStorage) Upload(c *models.ReqContext) response.Response {
-	// 32 MB is the default used by FormFile()
+	type rspInfo struct {
-	if err := c.Req.ParseMultipartForm(32 << 20); err != nil {
+		Message string `json:"message,omitempty"`
-		return response.Error(400, "error in parsing form", err)
+		Path    string `json:"path,omitempty"`
+		Count   int    `json:"count,omitempty"`
+		Bytes   int    `json:"bytes,omitempty"`
+		Error   bool   `json:"err,omitempty"`
 	}
+	rsp := &rspInfo{Message: "uploaded"}
+
 	c.Req.Body = http.MaxBytesReader(c.Resp, c.Req.Body, MAX_UPLOAD_SIZE)
 	if err := c.Req.ParseMultipartForm(MAX_UPLOAD_SIZE); err != nil {
-		msg := fmt.Sprintf("Please limit file uploaded under %s", util.ByteCountSI(MAX_UPLOAD_SIZE))
+		rsp.Message = fmt.Sprintf("Please limit file uploaded under %s", util.ByteCountSI(MAX_UPLOAD_SIZE))
-		return response.Error(400, msg, err)
+		rsp.Error = true
-	}
+		return response.JSON(400, rsp)
-
+	}
-	files := c.Req.MultipartForm.File["file"]
+	message := getMultipartFormValue(c.Req, "message")
-	if len(files) != 1 {
+	overwriteExistingFile := getMultipartFormValue(c.Req, "overwriteExistingFile") != "false" // must explicitly overwrite
-		return response.JSON(400, map[string]interface{}{
+	folder := getMultipartFormValue(c.Req, "folder")
-			"message": "please upload files one at a time",
+
-			"err":     true,
+	for k, fileHeaders := range c.Req.MultipartForm.File {
-		})
+		path := getMultipartFormValue(c.Req, k+".path") // match the path with a file
-	}
+		if len(fileHeaders) > 1 {
-
+			path = ""
-	folder, ok := getMultipartFormValue(c.Req, "folder")
+		}
-	if !ok || folder == "" {
+		if path == "" && folder == "" {
-		return response.JSON(400, map[string]interface{}{
+			rsp.Message = "please specify the upload folder or full path"
-			"message": "please specify the upload folder",
+			rsp.Error = true
-			"err":     true,
+			return response.JSON(400, rsp)
-		})
-	}
-	overwriteExistingFile, _ := getMultipartFormValue(c.Req, "overwriteExistingFile")
-
-	fileHeader := files[0]
-	if fileHeader.Size > MAX_UPLOAD_SIZE {
-		return errFileTooBig
 		}
 
+		for _, fileHeader := range fileHeaders {
 			// restrict file size based on file size
 			// open each file to copy contents
 			file, err := fileHeader.Open()
@@ -104,40 +101,45 @@ func (s *httpStorage) Upload(c *models.ReqContext) response.Response {
 				return response.Error(500, "Internal Server Error", err)
 			}
 
-	if (len(data)) > MAX_UPLOAD_SIZE {
+			if path == "" {
-		return errFileTooBig
+				path = folder + "/" + fileHeader.Filename
 			}
 
-	path := folder + "/" + fileHeader.Filename
+			entityType := EntityTypeJSON
-
 			mimeType := http.DetectContentType(data)
+			if strings.HasPrefix(mimeType, "image") {
+				entityType = EntityTypeImage
+			}
 
 			err = s.store.Upload(c.Req.Context(), c.SignedInUser, &UploadRequest{
 				Contents:              data,
 				MimeType:              mimeType,
-		EntityType:            EntityTypeImage,
+				EntityType:            entityType,
 				Path:                  path,
-		OverwriteExistingFile: overwriteExistingFile == "true",
+				OverwriteExistingFile: overwriteExistingFile,
+				Properties: map[string]string{
+					"message": message, // the commit/changelog entry
+				},
 			})
 
 			if err != nil {
 				return response.Error(UploadErrorToStatusCode(err), err.Error(), err)
 			}
+			rsp.Count++
+			rsp.Bytes += len(data)
+			rsp.Path = path
+		}
+	}
 
-	return response.JSON(200, map[string]interface{}{
+	return response.JSON(200, rsp)
-		"message": "Uploaded successfully",
-		"path":    path,
-		"file":    fileHeader.Filename,
-		"err":     true,
-	})
 }
 
-func getMultipartFormValue(req *http.Request, key string) (string, bool) {
+func getMultipartFormValue(req *http.Request, key string) string {
 	v, ok := req.MultipartForm.Value[key]
 	if !ok || len(v) != 1 {
-		return "", false
+		return ""
 	}
-	return v[0], ok
+	return v[0]
 }
 
 func (s *httpStorage) Read(c *models.ReqContext) response.Response {

pkg/services/store/service.go

@@ -4,7 +4,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"strings"
+	"os"
+	"path/filepath"
 
 	"github.com/grafana/grafana/pkg/infra/filestorage"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -25,9 +26,8 @@ var ErrValidationFailed = errors.New("request validation failed")
 var ErrFileAlreadyExists = errors.New("file exists")
 
 const RootPublicStatic = "public-static"
-const RootResources = "resources"
 
-const MAX_UPLOAD_SIZE = 3 * 1024 * 1024 // 3MB
+const MAX_UPLOAD_SIZE = 1 * 1024 * 1024 // 3MB
 
 type DeleteFolderCmd struct {
 	Path  string `json:"path"`
@@ -85,11 +85,26 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles,
 			setDescription("Access files from the static public files"),
 	}
 
+	// Development dashboards
+	if setting.Env != setting.Prod {
+		devenv := filepath.Join(cfg.StaticRootPath, "..", "devenv")
+		if _, err := os.Stat(devenv); !os.IsNotExist(err) {
+			// path/to/whatever exists
+			s := newDiskStorage("devenv", "Development Environment", &StorageLocalDiskConfig{
+				Path: devenv,
+				Roots: []string{
+					"/dev-dashboards/",
+				},
+			}).setReadOnly(false).setDescription("Explore files within the developer environment directly")
+			globalRoots = append(globalRoots, s)
+		}
+	}
+
 	initializeOrgStorages := func(orgId int64) []storageRuntime {
 		storages := make([]storageRuntime, 0)
 		if features.IsEnabled(featuremgmt.FlagStorageLocalUpload) {
 			storages = append(storages,
-				newSQLStorage(RootResources,
+				newSQLStorage("resources",
 					"Resources",
 					&StorageSQLConfig{orgId: orgId}, sql).
 					setBuiltin(true).
@@ -155,22 +170,16 @@ type UploadRequest struct {
 	OverwriteExistingFile bool
 }
 
-func storageSupportsMutatingOperations(path string) bool {
-	// TODO: this is temporary - make it rbac-driven
-	return strings.HasPrefix(path, RootResources+"/") || path == RootResources
-}
-
 func (s *standardStorageService) Upload(ctx context.Context, user *models.SignedInUser, req *UploadRequest) error {
-	upload, _ := s.tree.getRoot(getOrgId(user), RootResources)
+	upload, storagePath := s.tree.getRoot(getOrgId(user), req.Path)
 	if upload == nil {
 		return ErrUploadFeatureDisabled
 	}
 
-	if !storageSupportsMutatingOperations(req.Path) {
+	if upload.Meta().ReadOnly {
 		return ErrUnsupportedStorage
 	}
 
-	storagePath := strings.TrimPrefix(req.Path, RootResources)
 	validationResult := s.validateUploadRequest(ctx, user, req, storagePath)
 	if !validationResult.ok {
 		grafanaStorageLogger.Warn("file upload validation failed", "filetype", req.MimeType, "path", req.Path, "reason", validationResult.reason)
@@ -186,7 +195,7 @@ func (s *standardStorageService) Upload(ctx context.Context, user *models.Signed
 	grafanaStorageLogger.Info("uploading a file", "filetype", req.MimeType, "path", req.Path)
 
 	if !req.OverwriteExistingFile {
-		file, err := upload.Get(ctx, storagePath)
+		file, err := upload.Store().Get(ctx, storagePath)
 		if err != nil {
 			grafanaStorageLogger.Error("failed while checking file existence", "err", err, "path", req.Path)
 			return ErrUploadInternalError
@@ -197,7 +206,7 @@ func (s *standardStorageService) Upload(ctx context.Context, user *models.Signed
 		}
 	}
 
-	if err := upload.Upsert(ctx, upsertCommand); err != nil {
+	if err := upload.Store().Upsert(ctx, upsertCommand); err != nil {
 		grafanaStorageLogger.Error("failed while uploading the file", "err", err, "path", req.Path)
 		return ErrUploadInternalError
 	}
@@ -206,34 +215,32 @@ func (s *standardStorageService) Upload(ctx context.Context, user *models.Signed
 }
 
 func (s *standardStorageService) DeleteFolder(ctx context.Context, user *models.SignedInUser, cmd *DeleteFolderCmd) error {
-	resources, _ := s.tree.getRoot(getOrgId(user), RootResources)
+	root, storagePath := s.tree.getRoot(getOrgId(user), cmd.Path)
-	if resources == nil {
+	if root == nil {
 		return fmt.Errorf("resources storage is not enabled")
 	}
 
-	if !storageSupportsMutatingOperations(cmd.Path) {
+	if root.Meta().ReadOnly {
 		return ErrUnsupportedStorage
 	}
 
-	storagePath := strings.TrimPrefix(cmd.Path, RootResources)
 	if storagePath == "" {
 		storagePath = filestorage.Delimiter
 	}
-	return resources.DeleteFolder(ctx, storagePath, &filestorage.DeleteFolderOptions{Force: true})
+	return root.Store().DeleteFolder(ctx, storagePath, &filestorage.DeleteFolderOptions{Force: true})
 }
 
 func (s *standardStorageService) CreateFolder(ctx context.Context, user *models.SignedInUser, cmd *CreateFolderCmd) error {
-	if !storageSupportsMutatingOperations(cmd.Path) {
+	root, storagePath := s.tree.getRoot(getOrgId(user), cmd.Path)
-		return ErrUnsupportedStorage
+	if root == nil {
-	}
-
-	resources, _ := s.tree.getRoot(getOrgId(user), RootResources)
-	if resources == nil {
 		return fmt.Errorf("resources storage is not enabled")
 	}
 
-	storagePath := strings.TrimPrefix(cmd.Path, RootResources)
+	if root.Meta().ReadOnly {
-	err := resources.CreateFolder(ctx, storagePath)
+		return ErrUnsupportedStorage
+	}
+
+	err := root.Store().CreateFolder(ctx, storagePath)
 	if err != nil {
 		return err
 	}
@@ -241,17 +248,16 @@ func (s *standardStorageService) CreateFolder(ctx context.Context, user *models.
 }
 
 func (s *standardStorageService) Delete(ctx context.Context, user *models.SignedInUser, path string) error {
-	if !storageSupportsMutatingOperations(path) {
+	root, storagePath := s.tree.getRoot(getOrgId(user), path)
-		return ErrUnsupportedStorage
+	if root == nil {
-	}
-
-	resources, _ := s.tree.getRoot(getOrgId(user), RootResources)
-	if resources == nil {
 		return fmt.Errorf("resources storage is not enabled")
 	}
 
-	storagePath := strings.TrimPrefix(path, RootResources)
+	if root.Meta().ReadOnly {
-	err := resources.Delete(ctx, storagePath)
+		return ErrUnsupportedStorage
+	}
+
+	err := root.Store().Delete(ctx, storagePath)
 	if err != nil {
 		return err
 	}

pkg/services/store/tree.go

@@ -11,7 +11,7 @@ import (
 
 type nestedTree struct {
 	rootsByOrgId map[int64][]storageRuntime
-	lookup       map[int64]map[string]filestorage.FileStorage
+	lookup       map[int64]map[string]storageRuntime
 
 	orgInitMutex          sync.Mutex
 	initializeOrgStorages func(orgId int64) []storageRuntime
@@ -21,10 +21,10 @@ var (
 	_ storageTree = (*nestedTree)(nil)
 )
 
-func asNameToFileStorageMap(storages []storageRuntime) map[string]filestorage.FileStorage {
+func asNameToFileStorageMap(storages []storageRuntime) map[string]storageRuntime {
-	lookup := make(map[string]filestorage.FileStorage)
+	lookup := make(map[string]storageRuntime)
 	for _, storage := range storages {
-		lookup[storage.Meta().Config.Prefix] = storage.Store()
+		lookup[storage.Meta().Config.Prefix] = storage
 	}
 	return lookup
 }
@@ -33,7 +33,7 @@ func (t *nestedTree) init() {
 	t.orgInitMutex.Lock()
 	defer t.orgInitMutex.Unlock()
 
-	t.lookup = make(map[int64]map[string]filestorage.FileStorage, len(t.rootsByOrgId))
+	t.lookup = make(map[int64]map[string]storageRuntime, len(t.rootsByOrgId))
 
 	for orgId, storages := range t.rootsByOrgId {
 		t.lookup[orgId] = asNameToFileStorageMap(storages)
@@ -50,7 +50,7 @@ func (t *nestedTree) assureOrgIsInitialized(orgId int64) {
 	}
 }
 
-func (t *nestedTree) getRoot(orgId int64, path string) (filestorage.FileStorage, string) {
+func (t *nestedTree) getRoot(orgId int64, path string) (storageRuntime, string) {
 	t.assureOrgIsInitialized(orgId)
 
 	if path == "" {
@@ -82,7 +82,7 @@ func (t *nestedTree) GetFile(ctx context.Context, orgId int64, path string) (*fi
 	if root == nil {
 		return nil, nil // not found (or not ready)
 	}
-	return root.Get(ctx, path)
+	return root.Store().Get(ctx, path)
 }
 
 func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string) (*StorageListFrame, error) {
@@ -146,7 +146,7 @@ func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string) (
 		return nil, nil // not found (or not ready)
 	}
 
-	listResponse, err := root.List(ctx, path, nil, &filestorage.ListOptions{
+	listResponse, err := root.Store().List(ctx, path, nil, &filestorage.ListOptions{
 		Recursive:   false,
 		WithFolders: true,
 		WithFiles:   true,

pkg/services/store/validate.go

@@ -74,15 +74,15 @@ func (s *standardStorageService) validateUploadRequest(ctx context.Context, user
 	}
 
 	switch req.EntityType {
+	case EntityTypeJSON:
+		fallthrough
 	case EntityTypeFolder:
 		fallthrough
 	case EntityTypeDashboard:
 		// TODO: add proper validation
-		var something interface{}
+		if !json.Valid(req.Contents) {
-		if err := json.Unmarshal(req.Contents, &something); err != nil {
+			return fail("invalid json")
-			return fail(err.Error())
 		}
-
 		return success()
 	case EntityTypeImage:
 		return s.validateImage(ctx, user, req)