FileStorage: Add upload form (#46749)

* move upload to http

* use storage from grafanads

* rever gomod changes

* fix test

* wip

* add upload func

* update upload func

* writing to uploads

* edit response from service

* use dropzone for UI

* modify response struct in service

* better read file

* set content type for svg

* restrict file types upload

* add test and clean up errors

* pass test

* fix backend lint errors

* limit type of files on FE

* add TODO for after merge

* rebase with storage changes

* comment out unused function

* update UI to not have 2 uploads

* only call upload on select

* use utils function to find * in path

* show preview on drag over

* not allowing upload of svg

* add preview to upload tab

* no console.log

* resolve conflicts

* refactor log line

* fix failing BE test

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
Co-authored-by: Artur Wierzbicki <artur.wierzbicki@grafana.com>

packages/grafana-ui/src/components/FileDropzone/FileDropzone.tsx

@@ -39,6 +39,7 @@ export interface FileDropzoneProps {
    * any list return null in the function.
    */
   fileListRenderer?: (file: DropzoneFile, removeFile: (file: DropzoneFile) => void) => ReactNode;
+  onFileRemove?: (file: DropzoneFile) => void;
 }
 
 export interface DropzoneFile {
@@ -50,7 +51,7 @@ export interface DropzoneFile {
   retryUpload?: () => void;
 }
 
-export function FileDropzone({ options, children, readAs, onLoad, fileListRenderer }: FileDropzoneProps) {
+export function FileDropzone({ options, children, readAs, onLoad, fileListRenderer, onFileRemove }: FileDropzoneProps) {
   const [files, setFiles] = useState<DropzoneFile[]>([]);
 
   const setFileProperty = useCallback(
@@ -137,9 +138,14 @@ export function FileDropzone({ options, children, readAs, onLoad, fileListRender
   const removeFile = (file: DropzoneFile) => {
     const newFiles = files.filter((f) => file.id !== f.id);
     setFiles(newFiles);
+    onFileRemove?.(file);
   };
 
-  const { getRootProps, getInputProps, isDragActive } = useDropzone({ ...options, useFsAccessApi: false, onDrop });
+  const { getRootProps, getInputProps, isDragActive } = useDropzone({
+    ...options,
+    useFsAccessApi: false,
+    onDrop,
+  });
   const theme = useTheme2();
   const styles = getStyles(theme, isDragActive);
   const fileList = files.map((file) => {

pkg/services/store/http.go

@@ -2,6 +2,7 @@ package store
 
 import (
 	"net/http"
+	"strings"
 
 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/models"
@@ -27,35 +28,53 @@ func ProvideHTTPService(store StorageService) HTTPStorageService {
 }
 
 func (s *httpStorage) Upload(c *models.ReqContext) response.Response {
-	action := "Upload"
+	// 32 MB is the default used by FormFile()
-	scope, path := getPathAndScope(c)
+	if err := c.Req.ParseMultipartForm(32 << 20); err != nil {
+		return response.Error(400, "error in parsing form", err)
+	}
+	const MAX_UPLOAD_SIZE = 1024 * 1024
+	c.Req.Body = http.MaxBytesReader(c.Resp, c.Req.Body, MAX_UPLOAD_SIZE)
+	if err := c.Req.ParseMultipartForm(MAX_UPLOAD_SIZE); err != nil {
+		return response.Error(400, "Please limit file uploaded under 1MB", err)
+	}
+	res, err := s.store.Upload(c.Req.Context(), c.SignedInUser, c.Req.MultipartForm)
 
-	return response.JSON(http.StatusOK, map[string]string{
+	if err != nil {
-		"action": action,
+		return response.Error(500, "Internal Server Error", err)
-		"scope":  scope,
+	}
-		"path":   path,
+
+	return response.JSON(res.statusCode, map[string]interface{}{
+		"message": res.message,
+		"path":    res.path,
+		"file":    res.fileName,
+		"err":     true,
 	})
 }
 
 func (s *httpStorage) Read(c *models.ReqContext) response.Response {
-	action := "Read"
+	// full path is api/storage/read/upload/example.jpg, but we only want the part after read
 	scope, path := getPathAndScope(c)
-
+	file, err := s.store.Read(c.Req.Context(), c.SignedInUser, scope+"/"+path)
-	return response.JSON(http.StatusOK, map[string]string{
+	if err != nil {
-		"action": action,
+		return response.Error(400, "cannot call read", err)
-		"scope":  scope,
+	}
-		"path":   path,
+	// set the correct content type for svg
-	})
+	if strings.HasSuffix(path, ".svg") {
+		c.Resp.Header().Set("Content-Type", "image/svg+xml")
+	}
+	return response.Respond(200, file.Contents)
 }
 
 func (s *httpStorage) Delete(c *models.ReqContext) response.Response {
-	action := "Delete"
+	// full path is api/storage/delete/upload/example.jpg, but we only want the part after upload
-	scope, path := getPathAndScope(c)
+	_, path := getPathAndScope(c)
-
+	err := s.store.Delete(c.Req.Context(), c.SignedInUser, "/"+path)
-	return response.JSON(http.StatusOK, map[string]string{
+	if err != nil {
-		"action": action,
+		return response.Error(400, "cannot call delete", err)
-		"scope":  scope,
+	}
-		"path":   path,
+	return response.JSON(200, map[string]string{
+		"message": "Removed file from storage",
+		"path":    path,
 	})
 }
 

pkg/services/store/service.go

@@ -2,6 +2,10 @@ package store
 
 import (
 	"context"
+	"fmt"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
 	"os"
 	"path/filepath"
 
@@ -18,7 +22,7 @@ import (
 var grafanaStorageLogger = log.New("grafanaStorageLogger")
 
 const RootPublicStatic = "public-static"
-
+const MAX_UPLOAD_SIZE = 1024 * 1024 // 1MB
 type StorageService interface {
 	registry.BackgroundService
 
@@ -27,6 +31,10 @@ type StorageService interface {
 
 	// Read raw file contents out of the store
 	Read(ctx context.Context, user *models.SignedInUser, path string) (*filestorage.File, error)
+
+	Upload(ctx context.Context, user *models.SignedInUser, form *multipart.Form) (*Response, error)
+
+	Delete(ctx context.Context, user *models.SignedInUser, path string) error
 }
 
 type standardStorageService struct {
@@ -34,6 +42,14 @@ type standardStorageService struct {
 	tree *nestedTree
 }
 
+type Response struct {
+	path       string
+	statusCode int
+	message    string
+	fileName   string
+	err        bool
+}
+
 func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, cfg *setting.Cfg) StorageService {
 	roots := []storageRuntime{
 		newDiskStorage(RootPublicStatic, "Public static files", &StorageLocalDiskConfig{
@@ -53,9 +69,14 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles,
 	_ = os.MkdirAll(storage, 0700)
 
 	if features.IsEnabled(featuremgmt.FlagStorageLocalUpload) {
+		upload := filepath.Join(storage, "upload")
+		_ = os.MkdirAll(upload, 0700)
 		roots = append(roots, newDiskStorage("upload", "Local file upload", &StorageLocalDiskConfig{
-			Path: filepath.Join(storage, "upload"),
+			Path: upload,
-		}))
+			Roots: []string{
+				"/",
+			},
+		}).setBuiltin(true))
 	}
 	s := newStandardStorageService(roots)
 	s.sql = sql
@@ -86,3 +107,85 @@ func (s *standardStorageService) Read(ctx context.Context, user *models.SignedIn
 	// TODO: permission check!
 	return s.tree.GetFile(ctx, path)
 }
+
+func isFileTypeValid(filetype string) bool {
+	if (filetype == "image/jpeg") || (filetype == "image/jpg") || (filetype == "image/gif") || (filetype == "image/png") || (filetype == "image/webp") {
+		return true
+	}
+	return false
+}
+
+func (s *standardStorageService) Upload(ctx context.Context, user *models.SignedInUser, form *multipart.Form) (*Response, error) {
+	response := Response{
+		path: "upload",
+	}
+	upload, _ := s.tree.getRoot("upload")
+	if upload == nil {
+		response.statusCode = 404
+		response.message = "upload feature is not enabled"
+		response.err = true
+		return &response, fmt.Errorf("upload feature is not enabled")
+	}
+
+	files := form.File["file"]
+	for _, fileHeader := range files {
+		// Restrict the size of each uploaded file to 1MB based on the header
+		if fileHeader.Size > MAX_UPLOAD_SIZE {
+			response.statusCode = 400
+			response.message = "The uploaded image is too big"
+			response.err = true
+			return &response, nil
+		}
+		// restrict file size based on file size
+		// open each file to copy contents
+		file, err := fileHeader.Open()
+		if err != nil {
+			return nil, err
+		}
+		err = file.Close()
+		if err != nil {
+			return nil, err
+		}
+		data, err := ioutil.ReadAll(file)
+		if err != nil {
+			return nil, err
+		}
+
+		filetype := http.DetectContentType(data)
+		path := "/" + fileHeader.Filename
+
+		grafanaStorageLogger.Info("uploading a file", "filetype", filetype, "path", path)
+		// only allow images to be uploaded
+		if !isFileTypeValid(filetype) {
+			return &Response{
+				statusCode: 400,
+				message:    "unsupported file type uploaded",
+				err:        true,
+			}, nil
+		}
+		err = upload.Upsert(ctx, &filestorage.UpsertFileCommand{
+			Path:     path,
+			Contents: data,
+		})
+		if err != nil {
+			return nil, err
+		}
+		response.message = "Uploaded successfully"
+		response.statusCode = 200
+		response.fileName = fileHeader.Filename
+		response.path = "upload/" + fileHeader.Filename
+	}
+	return &response, nil
+}
+
+func (s *standardStorageService) Delete(ctx context.Context, user *models.SignedInUser, path string) error {
+	upload, _ := s.tree.getRoot("upload")
+	if upload == nil {
+		return fmt.Errorf("upload feature is not enabled")
+	}
+	err := upload.Delete(ctx, path)
+	if err != nil {
+		return err
+	}
+	return nil
+}

pkg/services/store/service_test.go

@@ -3,12 +3,17 @@ package store
 import (
 	"bytes"
 	"context"
+	"mime/multipart"
+	"os"
 	"path"
 	"path/filepath"
 	"testing"
 
 	"github.com/grafana/grafana-plugin-sdk-go/experimental"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb/testdatasource"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -45,3 +50,18 @@ func TestListFiles(t *testing.T) {
 	err = experimental.CheckGoldenFrame(path.Join("testdata", "public_testdata_js_libraries.golden.txt"), frame, true)
 	require.NoError(t, err)
 }
+
+func TestUpload(t *testing.T) {
+	features := featuremgmt.WithFeatures(featuremgmt.FlagStorageLocalUpload)
+	path, err := os.Getwd()
+	require.NoError(t, err)
+	cfg := &setting.Cfg{AppURL: "http://localhost:3000/", DataPath: path}
+	s := ProvideService(nil, features, cfg)
+	testForm := &multipart.Form{
+		Value: map[string][]string{},
+		File:  map[string][]*multipart.FileHeader{},
+	}
+	res, err := s.Upload(context.Background(), nil, testForm)
+	require.NoError(t, err)
+	assert.Equal(t, res.path, "upload")
+}

public/app/features/dimensions/editors/FileUploader.tsx

@@ -0,0 +1,132 @@
+import { css } from '@emotion/css';
+import React, { Dispatch, SetStateAction, useState } from 'react';
+import SVG from 'react-inlinesvg';
+
+import { GrafanaTheme2 } from '@grafana/data';
+import { FileDropzone, useStyles2, Button, DropzoneFile, Field } from '@grafana/ui';
+
+import { MediaType } from '../types';
+interface Props {
+  setFormData: Dispatch<SetStateAction<FormData>>;
+  mediaType: MediaType;
+  setUpload: Dispatch<SetStateAction<boolean>>;
+  newValue: string;
+  error: ErrorResponse;
+}
+interface ErrorResponse {
+  message: string;
+}
+export function FileDropzoneCustomChildren({ secondaryText = 'Drag and drop here or browse' }) {
+  const styles = useStyles2(getStyles);
+
+  return (
+    <div className={styles.iconWrapper}>
+      <small className={styles.small}>{secondaryText}</small>
+      <Button type="button" icon="upload">
+        Upload
+      </Button>
+    </div>
+  );
+}
+export const FileUploader = ({ mediaType, setFormData, setUpload, error }: Props) => {
+  const styles = useStyles2(getStyles);
+  const [dropped, setDropped] = useState<boolean>(false);
+  const [file, setFile] = useState<string>('');
+
+  const Preview = () => (
+    <Field label="Preview">
+      <div className={styles.iconPreview}>
+        {mediaType === MediaType.Icon && <SVG src={file} className={styles.img} />}
+        {mediaType === MediaType.Image && <img src={file} className={styles.img} />}
+      </div>
+    </Field>
+  );
+
+  const onFileRemove = (file: DropzoneFile) => {
+    fetch(`/api/storage/delete/upload/${file.file.name}`, {
+      method: 'DELETE',
+    }).catch((error) => console.error('cannot delete file', error));
+  };
+
+  const acceptableFiles =
+    mediaType === 'icon' ? 'image/svg+xml' : 'image/jpeg,image/png,image/gif,image/png, image/webp';
+  return (
+    <FileDropzone
+      readAs="readAsBinaryString"
+      onFileRemove={onFileRemove}
+      options={{
+        accept: acceptableFiles,
+        multiple: false,
+        onDrop: (acceptedFiles: File[]) => {
+          let formData = new FormData();
+          formData.append('file', acceptedFiles[0]);
+          setFile(URL.createObjectURL(acceptedFiles[0]));
+          setDropped(true);
+          setFormData(formData);
+          setUpload(true);
+        },
+      }}
+    >
+      {error.message !== '' && dropped ? (
+        <p>{error.message}</p>
+      ) : dropped ? (
+        <Preview />
+      ) : (
+        <FileDropzoneCustomChildren />
+      )}
+    </FileDropzone>
+  );
+};
+
+function getStyles(theme: GrafanaTheme2, isDragActive?: boolean) {
+  return {
+    container: css`
+      display: flex;
+      flex-direction: column;
+      width: 100%;
+    `,
+    dropzone: css`
+      display: flex;
+      flex: 1;
+      flex-direction: column;
+      align-items: center;
+      padding: ${theme.spacing(6)};
+      border-radius: 2px;
+      border: 2px dashed ${theme.colors.border.medium};
+      background-color: ${isDragActive ? theme.colors.background.secondary : theme.colors.background.primary};
+      cursor: pointer;
+    `,
+    iconWrapper: css`
+      display: flex;
+      flex-direction: column;
+      align-items: center;
+    `,
+    acceptMargin: css`
+      margin: ${theme.spacing(2, 0, 1)};
+    `,
+    small: css`
+      color: ${theme.colors.text.secondary};
+      margin-bottom: ${theme.spacing(2)};
+    `,
+    iconContainer: css`
+      display: flex;
+      flex-direction: column;
+      width: 80%;
+      align-items: center;
+      align-self: center;
+    `,
+    iconPreview: css`
+      width: 238px;
+      height: 198px;
+      border: 1px solid ${theme.colors.border.medium};
+      display: flex;
+      align-items: center;
+      justify-content: center;
+    `,
+    img: css`
+      width: 147px;
+      height: 147px;
+      fill: ${theme.colors.text.primary};
+    `,
+  };
+}

public/app/features/dimensions/editors/ResourcePickerPopover.tsx

@@ -5,10 +5,13 @@ import { useOverlay } from '@react-aria/overlays';
 import React, { createRef, useState } from 'react';
 
 import { GrafanaTheme2 } from '@grafana/data';
+import { getBackendSrv } from '@grafana/runtime';
 import { Button, ButtonGroup, useStyles2 } from '@grafana/ui';
+import { config } from 'app/core/config';
 
 import { MediaType, PickerTabType, ResourceFolderName } from '../types';
 
+import { FileUploader } from './FileUploader';
 import { FolderPickerTab } from './FolderPickerTab';
 import { URLPickerTab } from './URLPickerTab';
 
@@ -19,6 +22,9 @@ interface Props {
   folderName: ResourceFolderName;
 }
 
+interface ErrorResponse {
+  message: string;
+}
 export const ResourcePickerPopover = (props: Props) => {
   const { value, onChange, mediaType, folderName } = props;
   const styles = useStyles2(getStyles);
@@ -33,6 +39,9 @@ export const ResourcePickerPopover = (props: Props) => {
 
   const [newValue, setNewValue] = useState<string>(value ?? '');
   const [activePicker, setActivePicker] = useState<PickerTabType>(PickerTabType.Folder);
+  const [formData, setFormData] = useState<FormData>(new FormData());
+  const [upload, setUpload] = useState<boolean>(false);
+  const [error, setError] = useState<ErrorResponse>({ message: '' });
 
   const getTabClassName = (tabName: PickerTabType) => {
     return `${styles.resourcePickerPopoverTab} ${activePicker === tabName && styles.resourcePickerPopoverActiveTab}`;
@@ -49,13 +58,23 @@ export const ResourcePickerPopover = (props: Props) => {
   );
 
   const renderURLPicker = () => <URLPickerTab newValue={newValue} setNewValue={setNewValue} mediaType={mediaType} />;
-
+  const renderUploader = () => (
+    <FileUploader
+      mediaType={mediaType}
+      setFormData={setFormData}
+      setUpload={setUpload}
+      newValue={newValue}
+      error={error}
+    />
+  );
   const renderPicker = () => {
     switch (activePicker) {
       case PickerTabType.Folder:
         return renderFolderPicker();
       case PickerTabType.URL:
         return renderURLPicker();
+      case PickerTabType.Upload:
+        return renderUploader();
       default:
         return renderFolderPicker();
     }
@@ -75,6 +94,16 @@ export const ResourcePickerPopover = (props: Props) => {
             <button className={getTabClassName(PickerTabType.URL)} onClick={() => setActivePicker(PickerTabType.URL)}>
               URL
             </button>
+            {config.featureToggles['storageLocalUpload'] ? (
+              <button
+                className={getTabClassName(PickerTabType.Upload)}
+                onClick={() => setActivePicker(PickerTabType.Upload)}
+              >
+                Upload
+              </button>
+            ) : (
+              ''
+            )}
           </div>
           <div className={styles.resourcePickerPopoverContent}>
             {renderPicker()}
@@ -85,7 +114,31 @@ export const ResourcePickerPopover = (props: Props) => {
               <Button
                 className={styles.button}
                 variant={newValue && newValue !== value ? 'primary' : 'secondary'}
-                onClick={() => onChange(newValue)}
+                onClick={() => {
+                  if (upload) {
+                    fetch('/api/storage/upload', {
+                      method: 'POST',
+                      body: formData,
+                    })
+                      .then((res) => {
+                        if (res.status >= 400) {
+                          res.json().then((data) => setError(data));
+                          return;
+                        } else {
+                          return res.json();
+                        }
+                      })
+                      .then((data) => {
+                        getBackendSrv()
+                          .get(`api/storage/read/${data.path}`)
+                          .then(() => setNewValue(`${config.appUrl}api/storage/read/${data.path}`))
+                          .then(() => onChange(`${config.appUrl}api/storage/read/${data.path}`));
+                      })
+                      .catch((err) => console.error(err));
+                  } else {
+                    onChange(newValue);
+                  }
+                }}
               >
                 Select
               </Button>

public/app/features/dimensions/types.ts

@@ -126,6 +126,7 @@ export enum MediaType {
 export enum PickerTabType {
   Folder = 'folder',
   URL = 'url',
+  Upload = 'upload',
 }
 
 export enum ResourcePickerSize {