FileStorage: Add upload form (#46749)

* move upload to http

* use storage from grafanads

* rever gomod changes

* fix test

* wip

* add upload func

* update upload func

* writing to uploads

* edit response from service

* use dropzone for UI

* modify response struct in service

* better read file

* set content type for svg

* restrict file types upload

* add test and clean up errors

* pass test

* fix backend lint errors

* limit type of files on FE

* add TODO for after merge

* rebase with storage changes

* comment out unused function

* update UI to not have 2 uploads

* only call upload on select

* use utils function to find * in path

* show preview on drag over

* not allowing upload of svg

* add preview to upload tab

* no console.log

* resolve conflicts

* refactor log line

* fix failing BE test

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
Co-authored-by: Artur Wierzbicki <artur.wierzbicki@grafana.com>

pkg/services/store/http.go

@@ -2,6 +2,7 @@ package store
 
 import (
 	"net/http"
+	"strings"
 
 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/models"
@@ -27,35 +28,53 @@ func ProvideHTTPService(store StorageService) HTTPStorageService {
 }
 
 func (s *httpStorage) Upload(c *models.ReqContext) response.Response {
-	action := "Upload"
-	scope, path := getPathAndScope(c)
+	// 32 MB is the default used by FormFile()
+	if err := c.Req.ParseMultipartForm(32 << 20); err != nil {
+		return response.Error(400, "error in parsing form", err)
+	}
+	const MAX_UPLOAD_SIZE = 1024 * 1024
+	c.Req.Body = http.MaxBytesReader(c.Resp, c.Req.Body, MAX_UPLOAD_SIZE)
+	if err := c.Req.ParseMultipartForm(MAX_UPLOAD_SIZE); err != nil {
+		return response.Error(400, "Please limit file uploaded under 1MB", err)
+	}
+	res, err := s.store.Upload(c.Req.Context(), c.SignedInUser, c.Req.MultipartForm)
 
-	return response.JSON(http.StatusOK, map[string]string{
-		"action": action,
-		"scope":  scope,
-		"path":   path,
+	if err != nil {
+		return response.Error(500, "Internal Server Error", err)
+	}
+
+	return response.JSON(res.statusCode, map[string]interface{}{
+		"message": res.message,
+		"path":    res.path,
+		"file":    res.fileName,
+		"err":     true,
 	})
 }
 
 func (s *httpStorage) Read(c *models.ReqContext) response.Response {
-	action := "Read"
+	// full path is api/storage/read/upload/example.jpg, but we only want the part after read
 	scope, path := getPathAndScope(c)
-
-	return response.JSON(http.StatusOK, map[string]string{
-		"action": action,
-		"scope":  scope,
-		"path":   path,
-	})
+	file, err := s.store.Read(c.Req.Context(), c.SignedInUser, scope+"/"+path)
+	if err != nil {
+		return response.Error(400, "cannot call read", err)
+	}
+	// set the correct content type for svg
+	if strings.HasSuffix(path, ".svg") {
+		c.Resp.Header().Set("Content-Type", "image/svg+xml")
+	}
+	return response.Respond(200, file.Contents)
 }
 
 func (s *httpStorage) Delete(c *models.ReqContext) response.Response {
-	action := "Delete"
-	scope, path := getPathAndScope(c)
-
-	return response.JSON(http.StatusOK, map[string]string{
-		"action": action,
-		"scope":  scope,
-		"path":   path,
+	// full path is api/storage/delete/upload/example.jpg, but we only want the part after upload
+	_, path := getPathAndScope(c)
+	err := s.store.Delete(c.Req.Context(), c.SignedInUser, "/"+path)
+	if err != nil {
+		return response.Error(400, "cannot call delete", err)
+	}
+	return response.JSON(200, map[string]string{
+		"message": "Removed file from storage",
+		"path":    path,
 	})
 }
 

pkg/services/store/service.go

@@ -2,6 +2,10 @@ package store
 
 import (
 	"context"
+	"fmt"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
 	"os"
 	"path/filepath"
 
@@ -18,7 +22,7 @@ import (
 var grafanaStorageLogger = log.New("grafanaStorageLogger")
 
 const RootPublicStatic = "public-static"
-
+const MAX_UPLOAD_SIZE = 1024 * 1024 // 1MB
 type StorageService interface {
 	registry.BackgroundService
 
@@ -27,6 +31,10 @@ type StorageService interface {
 
 	// Read raw file contents out of the store
 	Read(ctx context.Context, user *models.SignedInUser, path string) (*filestorage.File, error)
+
+	Upload(ctx context.Context, user *models.SignedInUser, form *multipart.Form) (*Response, error)
+
+	Delete(ctx context.Context, user *models.SignedInUser, path string) error
 }
 
 type standardStorageService struct {
@@ -34,6 +42,14 @@ type standardStorageService struct {
 	tree *nestedTree
 }
 
+type Response struct {
+	path       string
+	statusCode int
+	message    string
+	fileName   string
+	err        bool
+}
+
 func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, cfg *setting.Cfg) StorageService {
 	roots := []storageRuntime{
 		newDiskStorage(RootPublicStatic, "Public static files", &StorageLocalDiskConfig{
@@ -53,9 +69,14 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles,
 	_ = os.MkdirAll(storage, 0700)
 
 	if features.IsEnabled(featuremgmt.FlagStorageLocalUpload) {
+		upload := filepath.Join(storage, "upload")
+		_ = os.MkdirAll(upload, 0700)
 		roots = append(roots, newDiskStorage("upload", "Local file upload", &StorageLocalDiskConfig{
-			Path: filepath.Join(storage, "upload"),
-		}))
+			Path: upload,
+			Roots: []string{
+				"/",
+			},
+		}).setBuiltin(true))
 	}
 	s := newStandardStorageService(roots)
 	s.sql = sql
@@ -86,3 +107,85 @@ func (s *standardStorageService) Read(ctx context.Context, user *models.SignedIn
 	// TODO: permission check!
 	return s.tree.GetFile(ctx, path)
 }
+
+func isFileTypeValid(filetype string) bool {
+	if (filetype == "image/jpeg") || (filetype == "image/jpg") || (filetype == "image/gif") || (filetype == "image/png") || (filetype == "image/webp") {
+		return true
+	}
+	return false
+}
+
+func (s *standardStorageService) Upload(ctx context.Context, user *models.SignedInUser, form *multipart.Form) (*Response, error) {
+	response := Response{
+		path: "upload",
+	}
+	upload, _ := s.tree.getRoot("upload")
+	if upload == nil {
+		response.statusCode = 404
+		response.message = "upload feature is not enabled"
+		response.err = true
+		return &response, fmt.Errorf("upload feature is not enabled")
+	}
+
+	files := form.File["file"]
+	for _, fileHeader := range files {
+		// Restrict the size of each uploaded file to 1MB based on the header
+		if fileHeader.Size > MAX_UPLOAD_SIZE {
+			response.statusCode = 400
+			response.message = "The uploaded image is too big"
+			response.err = true
+			return &response, nil
+		}
+		// restrict file size based on file size
+		// open each file to copy contents
+		file, err := fileHeader.Open()
+		if err != nil {
+			return nil, err
+		}
+		err = file.Close()
+		if err != nil {
+			return nil, err
+		}
+		data, err := ioutil.ReadAll(file)
+		if err != nil {
+			return nil, err
+		}
+
+		filetype := http.DetectContentType(data)
+		path := "/" + fileHeader.Filename
+
+		grafanaStorageLogger.Info("uploading a file", "filetype", filetype, "path", path)
+		// only allow images to be uploaded
+		if !isFileTypeValid(filetype) {
+			return &Response{
+				statusCode: 400,
+				message:    "unsupported file type uploaded",
+				err:        true,
+			}, nil
+		}
+		err = upload.Upsert(ctx, &filestorage.UpsertFileCommand{
+			Path:     path,
+			Contents: data,
+		})
+		if err != nil {
+			return nil, err
+		}
+		response.message = "Uploaded successfully"
+		response.statusCode = 200
+		response.fileName = fileHeader.Filename
+		response.path = "upload/" + fileHeader.Filename
+	}
+	return &response, nil
+}
+
+func (s *standardStorageService) Delete(ctx context.Context, user *models.SignedInUser, path string) error {
+	upload, _ := s.tree.getRoot("upload")
+	if upload == nil {
+		return fmt.Errorf("upload feature is not enabled")
+	}
+	err := upload.Delete(ctx, path)
+	if err != nil {
+		return err
+	}
+	return nil
+}

pkg/services/store/service_test.go

@@ -3,12 +3,17 @@ package store
 import (
 	"bytes"
 	"context"
+	"mime/multipart"
+	"os"
 	"path"
 	"path/filepath"
 	"testing"
 
 	"github.com/grafana/grafana-plugin-sdk-go/experimental"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb/testdatasource"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -45,3 +50,18 @@ func TestListFiles(t *testing.T) {
 	err = experimental.CheckGoldenFrame(path.Join("testdata", "public_testdata_js_libraries.golden.txt"), frame, true)
 	require.NoError(t, err)
 }
+
+func TestUpload(t *testing.T) {
+	features := featuremgmt.WithFeatures(featuremgmt.FlagStorageLocalUpload)
+	path, err := os.Getwd()
+	require.NoError(t, err)
+	cfg := &setting.Cfg{AppURL: "http://localhost:3000/", DataPath: path}
+	s := ProvideService(nil, features, cfg)
+	testForm := &multipart.Form{
+		Value: map[string][]string{},
+		File:  map[string][]*multipart.FileHeader{},
+	}
+	res, err := s.Upload(context.Background(), nil, testForm)
+	require.NoError(t, err)
+	assert.Equal(t, res.path, "upload")
+}