IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Storage: refactor readonly support (#52127)

Browse Source
This commit is contained in:
Ryan McKinley 2022-07-13 10:15:25 -07:00 committed by GitHub
parent 074bcf8599
commit ab6cf9e94d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 129 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/services/store/entity_events.go
View File

@ -26,6 +26,7 @@ const (
EntityTypeDashboard EntityType = "dashboard"
EntityTypeFolder EntityType = "folder"
EntityTypeImage EntityType = "image"
EntityTypeJSON EntityType = "json"
)
// CreateDatabaseEntityId creates entityId for entities stored in the existing SQL tables

148
pkg/services/store/http.go
View File

@ -15,8 +15,6 @@ import (
"github.com/grafana/grafana/pkg/web"
)
var errFileTooBig = response.Error(400, "Please limit file uploaded under 1MB", errors.New("file is too big"))
// HTTPStorageService passes raw HTTP requests to a well typed storage service
type HTTPStorageService interface {
List(c *models.ReqContext) response.Response
@ -57,87 +55,91 @@ func UploadErrorToStatusCode(err error) int {
}
func (s *httpStorage) Upload(c *models.ReqContext) response.Response {
// 32 MB is the default used by FormFile()
if err := c.Req.ParseMultipartForm(32 << 20); err != nil {
return response.Error(400, "error in parsing form", err)
type rspInfo struct {
Message string `json:"message,omitempty"`
Path string `json:"path,omitempty"`
Count int `json:"count,omitempty"`
Bytes int `json:"bytes,omitempty"`
Error bool `json:"err,omitempty"`
}
rsp := &rspInfo{Message: "uploaded"}
c.Req.Body = http.MaxBytesReader(c.Resp, c.Req.Body, MAX_UPLOAD_SIZE)
if err := c.Req.ParseMultipartForm(MAX_UPLOAD_SIZE); err != nil {
msg := fmt.Sprintf("Please limit file uploaded under %s", util.ByteCountSI(MAX_UPLOAD_SIZE))
return response.Error(400, msg, err)
rsp.Message = fmt.Sprintf("Please limit file uploaded under %s", util.ByteCountSI(MAX_UPLOAD_SIZE))
rsp.Error = true
return response.JSON(400, rsp)
}
message := getMultipartFormValue(c.Req, "message")
overwriteExistingFile := getMultipartFormValue(c.Req, "overwriteExistingFile") != "false" // must explicitly overwrite
folder := getMultipartFormValue(c.Req, "folder")
for k, fileHeaders := range c.Req.MultipartForm.File {
path := getMultipartFormValue(c.Req, k+".path") // match the path with a file
if len(fileHeaders) > 1 {
path = ""
}
if path == "" && folder == "" {
rsp.Message = "please specify the upload folder or full path"
rsp.Error = true
return response.JSON(400, rsp)
}
for _, fileHeader := range fileHeaders {
// restrict file size based on file size
// open each file to copy contents
file, err := fileHeader.Open()
if err != nil {
return response.Error(500, "Internal Server Error", err)
}
err = file.Close()
if err != nil {
return response.Error(500, "Internal Server Error", err)
}
data, err := ioutil.ReadAll(file)
if err != nil {
return response.Error(500, "Internal Server Error", err)
}
if path == "" {
path = folder + "/" + fileHeader.Filename
}
entityType := EntityTypeJSON
mimeType := http.DetectContentType(data)
if strings.HasPrefix(mimeType, "image") {
entityType = EntityTypeImage
}
err = s.store.Upload(c.Req.Context(), c.SignedInUser, &UploadRequest{
Contents: data,
MimeType: mimeType,
EntityType: entityType,
Path: path,
OverwriteExistingFile: overwriteExistingFile,
Properties: map[string]string{
"message": message, // the commit/changelog entry
},
})
if err != nil {
return response.Error(UploadErrorToStatusCode(err), err.Error(), err)
}
rsp.Count++
rsp.Bytes += len(data)
rsp.Path = path
}
}
files := c.Req.MultipartForm.File["file"]
if len(files) != 1 {
return response.JSON(400, map[string]interface{}{
"message": "please upload files one at a time",
"err": true,
})
}
folder, ok := getMultipartFormValue(c.Req, "folder")
if !ok || folder == "" {
return response.JSON(400, map[string]interface{}{
"message": "please specify the upload folder",
"err": true,
})
}
overwriteExistingFile, _ := getMultipartFormValue(c.Req, "overwriteExistingFile")
fileHeader := files[0]
if fileHeader.Size > MAX_UPLOAD_SIZE {
return errFileTooBig
}
// restrict file size based on file size
// open each file to copy contents
file, err := fileHeader.Open()
if err != nil {
return response.Error(500, "Internal Server Error", err)
}
err = file.Close()
if err != nil {
return response.Error(500, "Internal Server Error", err)
}
data, err := ioutil.ReadAll(file)
if err != nil {
return response.Error(500, "Internal Server Error", err)
}
if (len(data)) > MAX_UPLOAD_SIZE {
return errFileTooBig
}
path := folder + "/" + fileHeader.Filename
mimeType := http.DetectContentType(data)
err = s.store.Upload(c.Req.Context(), c.SignedInUser, &UploadRequest{
Contents: data,
MimeType: mimeType,
EntityType: EntityTypeImage,
Path: path,
OverwriteExistingFile: overwriteExistingFile == "true",
})
if err != nil {
return response.Error(UploadErrorToStatusCode(err), err.Error(), err)
}
return response.JSON(200, map[string]interface{}{
"message": "Uploaded successfully",
"path": path,
"file": fileHeader.Filename,
"err": true,
})
return response.JSON(200, rsp)
}
func getMultipartFormValue(req *http.Request, key string) (string, bool) {
func getMultipartFormValue(req *http.Request, key string) string {
v, ok := req.MultipartForm.Value[key]
if !ok || len(v) != 1 {
return "", false
return ""
}
return v[0], ok
return v[0]
}
func (s *httpStorage) Read(c *models.ReqContext) response.Response {

76
pkg/services/store/service.go
View File

@ -4,7 +4,8 @@ import (
"context"
"errors"
"fmt"
"strings"
"os"
"path/filepath"
"github.com/grafana/grafana/pkg/infra/filestorage"
"github.com/grafana/grafana/pkg/infra/log"
@ -25,9 +26,8 @@ var ErrValidationFailed = errors.New("request validation failed")
var ErrFileAlreadyExists = errors.New("file exists")
const RootPublicStatic = "public-static"
const RootResources = "resources"
const MAX_UPLOAD_SIZE = 3 * 1024 * 1024 // 3MB
const MAX_UPLOAD_SIZE = 1 * 1024 * 1024 // 3MB
type DeleteFolderCmd struct {
Path string `json:"path"`
@ -85,11 +85,26 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles,
setDescription("Access files from the static public files"),
}
// Development dashboards
if setting.Env != setting.Prod {
devenv := filepath.Join(cfg.StaticRootPath, "..", "devenv")
if _, err := os.Stat(devenv); !os.IsNotExist(err) {
// path/to/whatever exists
s := newDiskStorage("devenv", "Development Environment", &StorageLocalDiskConfig{
Path: devenv,
Roots: []string{
"/dev-dashboards/",
},
}).setReadOnly(false).setDescription("Explore files within the developer environment directly")
globalRoots = append(globalRoots, s)
}
}
initializeOrgStorages := func(orgId int64) []storageRuntime {
storages := make([]storageRuntime, 0)
if features.IsEnabled(featuremgmt.FlagStorageLocalUpload) {
storages = append(storages,
newSQLStorage(RootResources,
newSQLStorage("resources",
"Resources",
&StorageSQLConfig{orgId: orgId}, sql).
setBuiltin(true).
@ -155,22 +170,16 @@ type UploadRequest struct {
OverwriteExistingFile bool
}
func storageSupportsMutatingOperations(path string) bool {
// TODO: this is temporary - make it rbac-driven
return strings.HasPrefix(path, RootResources+"/") || path == RootResources
}
func (s *standardStorageService) Upload(ctx context.Context, user *models.SignedInUser, req *UploadRequest) error {
upload, _ := s.tree.getRoot(getOrgId(user), RootResources)
upload, storagePath := s.tree.getRoot(getOrgId(user), req.Path)
if upload == nil {
return ErrUploadFeatureDisabled
}
if !storageSupportsMutatingOperations(req.Path) {
if upload.Meta().ReadOnly {
return ErrUnsupportedStorage
}
storagePath := strings.TrimPrefix(req.Path, RootResources)
validationResult := s.validateUploadRequest(ctx, user, req, storagePath)
if !validationResult.ok {
grafanaStorageLogger.Warn("file upload validation failed", "filetype", req.MimeType, "path", req.Path, "reason", validationResult.reason)
@ -186,7 +195,7 @@ func (s *standardStorageService) Upload(ctx context.Context, user *models.Signed
grafanaStorageLogger.Info("uploading a file", "filetype", req.MimeType, "path", req.Path)
if !req.OverwriteExistingFile {
file, err := upload.Get(ctx, storagePath)
file, err := upload.Store().Get(ctx, storagePath)
if err != nil {
grafanaStorageLogger.Error("failed while checking file existence", "err", err, "path", req.Path)
return ErrUploadInternalError
@ -197,7 +206,7 @@ func (s *standardStorageService) Upload(ctx context.Context, user *models.Signed
}
}
if err := upload.Upsert(ctx, upsertCommand); err != nil {
if err := upload.Store().Upsert(ctx, upsertCommand); err != nil {
grafanaStorageLogger.Error("failed while uploading the file", "err", err, "path", req.Path)
return ErrUploadInternalError
}
@ -206,34 +215,32 @@ func (s *standardStorageService) Upload(ctx context.Context, user *models.Signed
}
func (s *standardStorageService) DeleteFolder(ctx context.Context, user *models.SignedInUser, cmd *DeleteFolderCmd) error {
resources, _ := s.tree.getRoot(getOrgId(user), RootResources)
if resources == nil {
root, storagePath := s.tree.getRoot(getOrgId(user), cmd.Path)
if root == nil {
return fmt.Errorf("resources storage is not enabled")
}
if !storageSupportsMutatingOperations(cmd.Path) {
if root.Meta().ReadOnly {
return ErrUnsupportedStorage
}
storagePath := strings.TrimPrefix(cmd.Path, RootResources)
if storagePath == "" {
storagePath = filestorage.Delimiter
}
return resources.DeleteFolder(ctx, storagePath, &filestorage.DeleteFolderOptions{Force: true})
return root.Store().DeleteFolder(ctx, storagePath, &filestorage.DeleteFolderOptions{Force: true})
}
func (s *standardStorageService) CreateFolder(ctx context.Context, user *models.SignedInUser, cmd *CreateFolderCmd) error {
if !storageSupportsMutatingOperations(cmd.Path) {
return ErrUnsupportedStorage
}
resources, _ := s.tree.getRoot(getOrgId(user), RootResources)
if resources == nil {
root, storagePath := s.tree.getRoot(getOrgId(user), cmd.Path)
if root == nil {
return fmt.Errorf("resources storage is not enabled")
}
storagePath := strings.TrimPrefix(cmd.Path, RootResources)
err := resources.CreateFolder(ctx, storagePath)
if root.Meta().ReadOnly {
return ErrUnsupportedStorage
}
err := root.Store().CreateFolder(ctx, storagePath)
if err != nil {
return err
}
@ -241,17 +248,16 @@ func (s *standardStorageService) CreateFolder(ctx context.Context, user *models.
}
func (s *standardStorageService) Delete(ctx context.Context, user *models.SignedInUser, path string) error {
if !storageSupportsMutatingOperations(path) {
return ErrUnsupportedStorage
}
resources, _ := s.tree.getRoot(getOrgId(user), RootResources)
if resources == nil {
root, storagePath := s.tree.getRoot(getOrgId(user), path)
if root == nil {
return fmt.Errorf("resources storage is not enabled")
}
storagePath := strings.TrimPrefix(path, RootResources)
err := resources.Delete(ctx, storagePath)
if root.Meta().ReadOnly {
return ErrUnsupportedStorage
}
err := root.Store().Delete(ctx, storagePath)
if err != nil {
return err
}

16
pkg/services/store/tree.go
View File

@ -11,7 +11,7 @@ import (
type nestedTree struct {
rootsByOrgId map[int64][]storageRuntime
lookup map[int64]map[string]filestorage.FileStorage
lookup map[int64]map[string]storageRuntime
orgInitMutex sync.Mutex
initializeOrgStorages func(orgId int64) []storageRuntime
@ -21,10 +21,10 @@ var (
_ storageTree = (*nestedTree)(nil)
)
func asNameToFileStorageMap(storages []storageRuntime) map[string]filestorage.FileStorage {
lookup := make(map[string]filestorage.FileStorage)
func asNameToFileStorageMap(storages []storageRuntime) map[string]storageRuntime {
lookup := make(map[string]storageRuntime)
for _, storage := range storages {
lookup[storage.Meta().Config.Prefix] = storage.Store()
lookup[storage.Meta().Config.Prefix] = storage
}
return lookup
}
@ -33,7 +33,7 @@ func (t *nestedTree) init() {
t.orgInitMutex.Lock()
defer t.orgInitMutex.Unlock()
t.lookup = make(map[int64]map[string]filestorage.FileStorage, len(t.rootsByOrgId))
t.lookup = make(map[int64]map[string]storageRuntime, len(t.rootsByOrgId))
for orgId, storages := range t.rootsByOrgId {
t.lookup[orgId] = asNameToFileStorageMap(storages)
@ -50,7 +50,7 @@ func (t *nestedTree) assureOrgIsInitialized(orgId int64) {
}
}
func (t *nestedTree) getRoot(orgId int64, path string) (filestorage.FileStorage, string) {
func (t *nestedTree) getRoot(orgId int64, path string) (storageRuntime, string) {
t.assureOrgIsInitialized(orgId)
if path == "" {
@ -82,7 +82,7 @@ func (t *nestedTree) GetFile(ctx context.Context, orgId int64, path string) (*fi
if root == nil {
return nil, nil // not found (or not ready)
}
return root.Get(ctx, path)
return root.Store().Get(ctx, path)
}
func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string) (*StorageListFrame, error) {
@ -146,7 +146,7 @@ func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string) (
return nil, nil // not found (or not ready)
}
listResponse, err := root.List(ctx, path, nil, &filestorage.ListOptions{
listResponse, err := root.Store().List(ctx, path, nil, &filestorage.ListOptions{
Recursive: false,
WithFolders: true,
WithFiles: true,

8
pkg/services/store/validate.go
View File

@ -74,15 +74,15 @@ func (s *standardStorageService) validateUploadRequest(ctx context.Context, user
}
switch req.EntityType {
case EntityTypeJSON:
fallthrough
case EntityTypeFolder:
fallthrough
case EntityTypeDashboard:
// TODO: add proper validation
var something interface{}
if err := json.Unmarshal(req.Contents, &something); err != nil {
return fail(err.Error())
if !json.Valid(req.Contents) {
return fail("invalid json")
}
return success()
case EntityTypeImage:
return s.validateImage(ctx, user, req)