IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Storage: Content root storage (#54929)

Browse Source 
Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
This commit is contained in:
Artur Wierzbicki 2022-09-13 00:34:46 +02:00 committed by GitHub
parent 89d94eeab2
commit e19f36649f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 538 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/infra/filestorage/filter.go
View File

@ -244,6 +244,6 @@ func (m allOfPathFilter) asSQLFilter() accesscontrol.SQLFilter {
}
}
func newAndPathFilter(filters ...PathFilter) PathFilter {
func NewAndPathFilter(filters ...PathFilter) PathFilter {
return &allOfPathFilter{filters: filters}
}

4
pkg/infra/filestorage/wrapper.go
View File

@ -240,7 +240,7 @@ func (b wrapper) listOptionsWithDefaults(options *ListOptions) *ListOptions {
var filter PathFilter
if options.Filter != nil {
filter = newAndPathFilter(b.filter, wrapPathFilter(options.Filter, b.rootFolder))
filter = NewAndPathFilter(b.filter, wrapPathFilter(options.Filter, b.rootFolder))
} else {
filter = b.filter
}
@ -283,7 +283,7 @@ func (b wrapper) deleteFolderOptionsWithDefaults(options *DeleteFolderOptions) *
var filter PathFilter
if options.AccessFilter != nil {
filter = newAndPathFilter(b.filter, wrapPathFilter(options.AccessFilter, b.rootFolder))
filter = NewAndPathFilter(b.filter, wrapPathFilter(options.AccessFilter, b.rootFolder))
} else {
filter = b.filter
}

11
pkg/services/store/config.go
View File

@ -99,11 +99,12 @@ func (c *GlobalStorageConfig) save() error {
}
type RootStorageConfig struct {
Type string `json:"type"`
Prefix string `json:"prefix"`
Name string `json:"name"`
Description string `json:"description"`
Disabled bool `json:"disabled,omitempty"`
Type string `json:"type"`
Prefix string `json:"prefix"`
UnderContentRoot bool `json:"underContentRoot"`
Name string `json:"name"`
Description string `json:"description"`
Disabled bool `json:"disabled,omitempty"`
// Depending on type, these will be configured
Disk *StorageLocalDiskConfig `json:"disk,omitempty"`

82
pkg/services/store/service.go
View File

@ -15,6 +15,7 @@ import (
"github.com/grafana/grafana/pkg/registry"
ac "github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/services/quota"
"github.com/grafana/grafana/pkg/services/sqlstore/db"
"github.com/grafana/grafana/pkg/services/user"
@ -34,6 +35,7 @@ var ErrOnlyDashboardSaveSupported = errors.New("only dashboard save is currently
const RootPublicStatic = "public-static"
const RootResources = "resources"
const RootContent = "content"
const RootDevenv = "devenv"
const RootSystem = "system"
@ -129,9 +131,10 @@ func ProvideService(
s := newDiskStorage(RootStorageMeta{
ReadOnly: false,
}, RootStorageConfig{
Prefix: RootDevenv,
Name: "Development Environment",
Description: "Explore files within the developer environment directly",
Prefix: RootDevenv,
UnderContentRoot: true,
Name: "Development Environment",
Description: "Explore files within the developer environment directly",
Disk: &StorageLocalDiskConfig{
Path: devenv,
Roots: []string{
@ -147,6 +150,9 @@ func ProvideService(
grafanaStorageLogger.Warn("Invalid root configuration", "cfg", root)
continue
}
// all externally-defined storages lie under the "content" root
root.UnderContentRoot = true
s, err := newStorage(root, filepath.Join(cfg.DataPath, "storage", "cache", root.Prefix))
if err != nil {
grafanaStorageLogger.Warn("error loading storage config", "error", err)
@ -159,23 +165,22 @@ func ProvideService(
initializeOrgStorages := func(orgId int64) []storageRuntime {
storages := make([]storageRuntime, 0)
storages = append(storages,
newSQLStorage(RootStorageMeta{
Builtin: true,
}, RootContent, "Content", "Content root", &StorageSQLConfig{}, sql, orgId, false))
// Custom upload files
storages = append(storages,
newSQLStorage(RootStorageMeta{
Builtin: true,
}, RootResources,
"Resources",
"Upload custom resource files",
&StorageSQLConfig{}, sql, orgId))
}, RootResources, "Resources", "Upload custom resource files", &StorageSQLConfig{}, sql, orgId, false))
// System settings
storages = append(storages,
newSQLStorage(RootStorageMeta{
Builtin: true,
}, RootSystem,
"System",
"Grafana system storage",
&StorageSQLConfig{}, sql, orgId))
}, RootSystem, "System", "Grafana system storage", &StorageSQLConfig{}, sql, orgId, false))
return storages
}
@ -215,6 +220,30 @@ func ProvideService(
}
}
if storageName == RootContent {
if user.OrgRole != org.RoleAdmin {
// read only
return map[string]filestorage.PathFilter{
ActionFilesRead: allowAllPathFilter,
ActionFilesWrite: denyAllPathFilter,
ActionFilesDelete: denyAllPathFilter,
}
}
// read/write for all except for devenv
writeFilter := filestorage.NewPathFilter(
[]string{filestorage.Delimiter}, // access to everything
nil,
[]string{filestorage.Delimiter + RootDevenv + filestorage.Delimiter}, // except devenv
[]string{filestorage.Delimiter + RootDevenv})
return map[string]filestorage.PathFilter{
ActionFilesRead: allowAllPathFilter,
ActionFilesWrite: writeFilter,
ActionFilesDelete: writeFilter,
}
}
if !user.IsGrafanaAdmin {
return nil
}
@ -248,6 +277,17 @@ func newStandardStorageService(
authService storageAuthService,
cfg *setting.Cfg,
) *standardStorageService {
prefixes := make(map[string]bool)
for _, root := range globalRoots {
currentPrefix := root.Meta().Config.Prefix
if _, ok := prefixes[currentPrefix]; ok {
panic("non-unique storage prefix: " + currentPrefix)
}
prefixes[currentPrefix] = true
}
rootsByOrgId := make(map[int64][]storageRuntime)
rootsByOrgId[ac.GlobalOrgID] = globalRoots
@ -384,6 +424,10 @@ func (s *standardStorageService) DeleteFolder(ctx context.Context, user *user.Si
return ErrUnsupportedStorage
}
if err := s.validateFolderNameDoesNotConflictWithNestedStorages(root, storagePath, user.OrgID); err != nil {
return err
}
if storagePath == "" {
storagePath = filestorage.Delimiter
}
@ -409,6 +453,10 @@ func (s *standardStorageService) CreateFolder(ctx context.Context, user *user.Si
return ErrUnsupportedStorage
}
if err := s.validateFolderNameDoesNotConflictWithNestedStorages(root, storagePath, user.OrgID); err != nil {
return err
}
err := root.Store().CreateFolder(ctx, storagePath)
if err != nil {
return err
@ -416,6 +464,18 @@ func (s *standardStorageService) CreateFolder(ctx context.Context, user *user.Si
return nil
}
func (s *standardStorageService) validateFolderNameDoesNotConflictWithNestedStorages(root storageRuntime, storagePath string, orgID int64) error {
if !root.Meta().Config.UnderContentRoot {
return nil
}
if storagePath == "" || storagePath == "/" {
return ErrValidationFailed
}
return nil
}
func (s *standardStorageService) Delete(ctx context.Context, user *user.SignedInUser, path string) error {
guardian := s.authService.newGuardian(ctx, user, getFirstSegment(path))
if !guardian.canDelete(path) {

310
pkg/services/store/service_test.go
View File

@ -3,12 +3,15 @@ package store
import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/grafana/grafana-plugin-sdk-go/experimental"
"github.com/grafana/grafana/pkg/infra/filestorage"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/quota/quotatest"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/services/user"
@ -29,6 +32,7 @@ var (
jpgBytes, _ = os.ReadFile("testdata/image.jpg")
svgBytes, _ = os.ReadFile("testdata/image.svg")
dummyUser = &user.SignedInUser{OrgID: 1}
globalUser = &user.SignedInUser{OrgID: 0}
allowAllAuthService = newStaticStorageAuthService(func(ctx context.Context, user *user.SignedInUser, storageName string) map[string]filestorage.PathFilter {
return map[string]filestorage.PathFilter{
ActionFilesDelete: allowAllPathFilter,
@ -101,13 +105,7 @@ func setupUploadStore(t *testing.T, authService storageAuthService) (StorageServ
t.Helper()
storageName := "resources"
mockStorage := &filestorage.MockFileStorage{}
sqlStorage := newSQLStorage(
RootStorageMeta{},
storageName, "Testing upload", "dummy descr",
&StorageSQLConfig{},
sqlstore.InitTestDB(t),
1, // orgID (prefix init)
)
sqlStorage := newSQLStorage(RootStorageMeta{}, storageName, "Testing upload", "dummy descr", &StorageSQLConfig{}, sqlstore.InitTestDB(t), 1, false)
sqlStorage.store = mockStorage
if authService == nil {
@ -255,3 +253,301 @@ func TestShouldNotUploadJpgDisguisedAsSvg(t *testing.T) {
})
require.ErrorIs(t, err, ErrValidationFailed)
}
func TestSetupWithNonUniqueStoragePrefixes(t *testing.T) {
prefix := "resources"
sqlStorage := newSQLStorage(RootStorageMeta{}, prefix, "Testing upload", "dummy descr", &StorageSQLConfig{}, sqlstore.InitTestDB(t), 1, false)
sqlStorage2 := newSQLStorage(RootStorageMeta{}, prefix, "Testing upload", "dummy descr", &StorageSQLConfig{}, sqlstore.InitTestDB(t), 1, false)
defer func() {
if r := recover(); r == nil {
t.Errorf("The setup should have panicked")
}
}()
newStandardStorageService(sqlstore.InitTestDB(t), []storageRuntime{sqlStorage, sqlStorage2}, func(orgId int64) []storageRuntime {
return make([]storageRuntime, 0)
}, allowAllAuthService, cfg)
}
func TestContentRootWithNestedStorage(t *testing.T) {
globalOrgID := int64(accesscontrol.GlobalOrgID)
db := sqlstore.InitTestDB(t)
orgedUser := &user.SignedInUser{OrgID: 1}
t.Helper()
mockContentFSApi := &filestorage.MockFileStorage{}
contentStorage := newSQLStorage(RootStorageMeta{}, RootContent, "Content root", "dummy descr", &StorageSQLConfig{}, db, globalOrgID, false)
contentStorage.store = mockContentFSApi
nestedRoot := "nested"
mockNestedFSApi := &filestorage.MockFileStorage{}
nestedStorage := newSQLStorage(RootStorageMeta{}, nestedRoot, "Nested root", "dummy descr", &StorageSQLConfig{}, db, globalOrgID, true)
nestedStorage.store = mockNestedFSApi
nestedOrgedRoot := "nestedOrged"
mockNestedOrgedFSApi := &filestorage.MockFileStorage{}
nestedOrgedStorage := newSQLStorage(RootStorageMeta{}, nestedOrgedRoot, "Nested root", "dummy descr", &StorageSQLConfig{}, db, globalOrgID, true)
nestedOrgedStorage.store = mockNestedOrgedFSApi
store := newStandardStorageService(sqlstore.InitTestDB(t), []storageRuntime{contentStorage, nestedStorage}, func(orgId int64) []storageRuntime {
return []storageRuntime{nestedOrgedStorage, contentStorage}
}, allowAllAuthService, cfg)
store.cfg = &GlobalStorageConfig{
AllowUnsanitizedSvgUpload: true,
}
store.quotaService = quotatest.NewQuotaServiceFake()
fileName := "file.jpg"
tests := []struct {
user *user.SignedInUser
name string
mockNestedFS *filestorage.MockFileStorage
nestedRoot string
}{
{
user: globalUser,
name: "global user, global nested storage",
mockNestedFS: mockNestedFSApi,
nestedRoot: nestedRoot,
},
{
user: orgedUser,
name: "non-global user, global nested storage",
mockNestedFS: mockNestedFSApi,
nestedRoot: nestedRoot,
},
{
user: orgedUser,
name: "non-global user, non-global nested storage",
mockNestedFS: mockNestedOrgedFSApi,
nestedRoot: nestedOrgedRoot,
},
}
for _, test := range tests {
t.Run(test.name+": Uploading a file under a /content/nested/.. should delegate to the nested storage", func(t *testing.T) {
test.mockNestedFS.On("Get", mock.Anything, filestorage.Delimiter+fileName, &filestorage.GetFileOptions{WithContents: false}).Return(nil, false, nil)
test.mockNestedFS.On("Upsert", mock.Anything, &filestorage.UpsertFileCommand{
Path: filestorage.Delimiter + fileName,
MimeType: "image/jpeg",
Contents: jpgBytes,
}).Return(nil)
mockContentFSApi.AssertNotCalled(t, "Get")
mockContentFSApi.AssertNotCalled(t, "Upsert")
err := store.Upload(context.Background(), test.user, &UploadRequest{
EntityType: EntityTypeImage,
Contents: jpgBytes,
Path: strings.Join([]string{RootContent, test.nestedRoot, fileName}, filestorage.Delimiter),
})
require.NoError(t, err)
})
t.Run(test.name+": Creating a /content/nested folder should fail", func(t *testing.T) {
mockContentFSApi.AssertNotCalled(t, "CreateFolder")
err := store.CreateFolder(context.Background(), test.user, &CreateFolderCmd{Path: RootContent + "/" + test.nestedRoot})
require.ErrorIs(t, err, ErrValidationFailed)
})
t.Run(test.name+": Deleting a /content/nested folder should fail", func(t *testing.T) {
mockContentFSApi.AssertNotCalled(t, "DeleteFolder")
err := store.DeleteFolder(context.Background(), test.user, &DeleteFolderCmd{Path: RootContent + "/" + test.nestedRoot})
require.ErrorIs(t, err, ErrValidationFailed)
})
t.Run(test.name+": Listing /content/nested should delegate to the nested root", func(t *testing.T) {
mockContentFSApi.AssertNotCalled(t, "List")
test.mockNestedFS.On(
"List",
mock.Anything,
"/",
mock.Anything,
mock.Anything,
).Return(&filestorage.ListResponse{
Files: []*filestorage.File{},
}, nil)
_, err := store.List(context.Background(), test.user, RootContent+"/"+test.nestedRoot)
require.NoError(t, err)
})
t.Run(test.name+": Listing a folder inside /content/nested/.. should delegate to the nested root", func(t *testing.T) {
mockContentFSApi.AssertNotCalled(t, "List")
test.mockNestedFS.On(
"List",
mock.Anything,
"/folder1/folder2",
mock.Anything,
mock.Anything,
).Return(&filestorage.ListResponse{
Files: []*filestorage.File{},
}, nil)
_, err := store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot, "folder1", "folder2"}, "/"))
require.NoError(t, err)
})
t.Run(test.name+": Listing outside of the nested storages should delegate to the content root", func(t *testing.T) {
test.mockNestedFS.AssertNotCalled(t, "List")
mockContentFSApi.On(
"List",
mock.Anything,
"/not-nested-content",
mock.Anything,
mock.Anything,
).Return(&filestorage.ListResponse{
Files: []*filestorage.File{},
}, nil)
mockContentFSApi.On(
"List",
mock.Anything,
"/a/b/c",
mock.Anything,
mock.Anything,
).Return(&filestorage.ListResponse{
Files: []*filestorage.File{},
}, nil)
mockContentFSApi.On(
"List",
mock.Anything,
fmt.Sprintf("/%sa", test.nestedRoot),
mock.Anything,
mock.Anything,
).Return(&filestorage.ListResponse{
Files: []*filestorage.File{},
}, nil)
mockContentFSApi.On(
"List",
mock.Anything,
fmt.Sprintf("/%sa/b", test.nestedRoot),
mock.Anything,
mock.Anything,
).Return(&filestorage.ListResponse{
Files: []*filestorage.File{},
}, nil)
_, err := store.List(context.Background(), test.user, strings.Join([]string{RootContent, "not-nested-content"}, "/"))
require.NoError(t, err)
_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, "a", "b", "c"}, "/"))
require.NoError(t, err)
_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot + "a"}, "/"))
require.NoError(t, err)
_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot + "a", "b"}, "/"))
require.NoError(t, err)
})
t.Run(test.name+": Uploading files outside of the nested storages should delegate to the content root", func(t *testing.T) {
test.mockNestedFS.AssertNotCalled(t, "Get")
test.mockNestedFS.AssertNotCalled(t, "Upsert")
// file at the root of the content root - /content/myFile.jpg
fileName := "myFile.jpg"
mockContentFSApi.On("Get", mock.Anything, "/"+fileName, &filestorage.GetFileOptions{WithContents: false}).Return(nil, false, nil)
mockContentFSApi.On("Upsert", mock.Anything, &filestorage.UpsertFileCommand{
Path: "/" + fileName,
MimeType: "image/jpeg",
Contents: jpgBytes,
}).Return(nil)
err := store.Upload(context.Background(), dummyUser, &UploadRequest{
EntityType: EntityTypeImage,
Contents: jpgBytes,
Path: strings.Join([]string{RootContent, fileName}, "/"),
})
require.NoError(t, err)
// file in the folder belonging to the content root storage - /content/nested/a/myFile.jpg
mockContentFSApi.On("Get", mock.Anything, "/a/"+fileName, &filestorage.GetFileOptions{WithContents: false}).Return(nil, false, nil)
mockContentFSApi.On("Upsert", mock.Anything, &filestorage.UpsertFileCommand{
Path: "/a/" + fileName,
MimeType: "image/jpeg",
Contents: jpgBytes,
}).Return(nil)
err = store.Upload(context.Background(), dummyUser, &UploadRequest{
EntityType: EntityTypeImage,
Contents: jpgBytes,
Path: strings.Join([]string{RootContent, "a", fileName}, "/"),
})
require.NoError(t, err)
})
t.Run(test.name+": Creating folders under /content/nested/.. should delegate to the nested roots", func(t *testing.T) {
mockContentFSApi.AssertNotCalled(t, "CreateFolder")
mockContentFSApi.AssertNotCalled(t, "DeleteFolder")
test.mockNestedFS.On("CreateFolder", mock.Anything, "/folder").Return(nil)
path := strings.Join([]string{RootContent, test.nestedRoot, "folder"}, "/")
err := store.CreateFolder(context.Background(), test.user, &CreateFolderCmd{Path: path})
require.NoError(t, err)
test.mockNestedFS.On("DeleteFolder", mock.Anything, "/folder", mock.Anything).Return(nil)
err = store.DeleteFolder(context.Background(), test.user, &DeleteFolderCmd{Path: path})
require.NoError(t, err)
})
t.Run(test.name+": Creating folders under outside of the nested storages should delegate to the content root", func(t *testing.T) {
test.mockNestedFS.AssertNotCalled(t, "CreateFolder")
test.mockNestedFS.AssertNotCalled(t, "DeleteFolder")
mockContentFSApi.On("CreateFolder", mock.Anything, "/folder").Return(nil)
path := strings.Join([]string{RootContent, "folder"}, "/")
err := store.CreateFolder(context.Background(), test.user, &CreateFolderCmd{Path: path})
require.NoError(t, err)
mockContentFSApi.On("DeleteFolder", mock.Anything, "/folder", mock.Anything).Return(nil)
err = store.DeleteFolder(context.Background(), test.user, &DeleteFolderCmd{Path: path})
require.NoError(t, err)
})
}
}
func TestShadowingExistingFolderByNestedContentRoot(t *testing.T) {
db := sqlstore.InitTestDB(t)
ctx := context.Background()
nestedStorage := newSQLStorage(RootStorageMeta{}, "nested", "Testing upload", "dummy descr", &StorageSQLConfig{}, db, accesscontrol.GlobalOrgID, true)
contentStorage := newSQLStorage(RootStorageMeta{}, RootContent, "Testing upload", "dummy descr", &StorageSQLConfig{}, db, accesscontrol.GlobalOrgID, false)
_, err := contentStorage.Write(ctx, &WriteValueRequest{
User: globalUser,
Path: "/nested/abc.jpg",
EntityType: EntityTypeImage,
Body: jpgBytes,
})
require.NoError(t, err)
store := newStandardStorageService(db, []storageRuntime{nestedStorage, contentStorage}, func(orgId int64) []storageRuntime { return make([]storageRuntime, 0) }, allowAllAuthService, cfg)
store.cfg = &GlobalStorageConfig{
AllowUnsanitizedSvgUpload: true,
}
resp, err := store.List(ctx, globalUser, "content/nested")
require.NoError(t, err)
require.NotNil(t, resp)
rowLen, err := resp.Frame.RowLen()
require.NoError(t, err)
require.Equal(t, 0, rowLen) // nested storage is empty
resp, err = store.List(ctx, globalUser, "content")
require.NoError(t, err)
require.NotNil(t, resp)
rowLen, err = resp.Frame.RowLen()
require.NoError(t, err)
require.Equal(t, 1, rowLen) // just a single "nested" folder
}

13
pkg/services/store/storage_sql.go
View File

@ -31,17 +31,18 @@ func getDbStoragePathPrefix(orgId int64, storageName string) string {
return filestorage.Join(fmt.Sprintf("%d", orgId), storageName+filestorage.Delimiter)
}
func newSQLStorage(meta RootStorageMeta, prefix string, name string, descr string, cfg *StorageSQLConfig, sql db.DB, orgId int64) *rootStorageSQL {
func newSQLStorage(meta RootStorageMeta, prefix string, name string, descr string, cfg *StorageSQLConfig, sql db.DB, orgId int64, underContentRoot bool) *rootStorageSQL {
if cfg == nil {
cfg = &StorageSQLConfig{}
}
meta.Config = RootStorageConfig{
Type: rootStorageTypeSQL,
Prefix: prefix,
Name: name,
Description: descr,
SQL: cfg,
Type: rootStorageTypeSQL,
Prefix: prefix,
Name: name,
Description: descr,
UnderContentRoot: underContentRoot,
SQL: cfg,
}
if prefix == "" {

105
pkg/services/store/tree.go
View File

@ -25,7 +25,13 @@ var (
func asNameToFileStorageMap(storages []storageRuntime) map[string]storageRuntime {
lookup := make(map[string]storageRuntime)
for _, storage := range storages {
lookup[storage.Meta().Config.Prefix] = storage
isUnderContentRoot := storage.Meta().Config.UnderContentRoot
prefix := storage.Meta().Config.Prefix
if !isUnderContentRoot {
lookup[prefix] = storage
} else {
lookup[fmt.Sprintf("%s/%s", RootContent, prefix)] = storage
}
}
return lookup
}
@ -61,12 +67,44 @@ func (t *nestedTree) getRoot(orgId int64, path string) (storageRuntime, string)
rootKey, path := splitFirstSegment(path)
root, ok := t.lookup[orgId][rootKey]
if ok && root != nil {
if root.Meta().Config.Prefix == RootContent && path != "" && path != "/" {
mountedKey, nestedPath := splitFirstSegment(path)
nestedLookupKey := rootKey + filestorage.Delimiter + mountedKey
nestedRoot, nestedOk := t.lookup[orgId][nestedLookupKey]
if nestedOk && nestedRoot != nil {
return nestedRoot, filestorage.Delimiter + nestedPath
}
if orgId != ac.GlobalOrgID {
globalRoot, globalOk := t.lookup[ac.GlobalOrgID][nestedLookupKey]
if globalOk && globalRoot != nil {
return globalRoot, filestorage.Delimiter + nestedPath
}
}
}
return root, filestorage.Delimiter + path
}
if orgId != ac.GlobalOrgID {
globalRoot, ok := t.lookup[ac.GlobalOrgID][rootKey]
if ok && globalRoot != nil {
if globalRoot.Meta().Config.Prefix == RootContent && path != "" && path != "/" {
mountedKey, nestedPath := splitFirstSegment(path)
nestedLookupKey := rootKey + filestorage.Delimiter + mountedKey
nestedRoot, nestedOk := t.lookup[orgId][nestedLookupKey]
if nestedOk && nestedRoot != nil {
return nestedRoot, filestorage.Delimiter + nestedPath
}
globalNestedRoot, globalOk := t.lookup[ac.GlobalOrgID][nestedLookupKey]
if globalOk && globalNestedRoot != nil {
return globalNestedRoot, filestorage.Delimiter + nestedPath
}
}
return globalRoot, filestorage.Delimiter + path
}
}
@ -90,12 +128,22 @@ func (t *nestedTree) GetFile(ctx context.Context, orgId int64, path string) (*fi
return file, err
}
func filterStoragesUnderContentRoot(storages []storageRuntime) []storageRuntime {
out := make([]storageRuntime, 0)
for _, s := range storages {
if s.Meta().Config.UnderContentRoot {
out = append(out, s)
}
}
return out
}
func (t *nestedTree) getStorages(orgId int64) []storageRuntime {
globalStorages := make([]storageRuntime, 0)
globalStorages = append(globalStorages, t.rootsByOrgId[ac.GlobalOrgID]...)
if orgId == ac.GlobalOrgID {
return globalStorages
return append(make([]storageRuntime, 0), globalStorages...)
}
orgPrefixes := make(map[string]bool)
@ -124,6 +172,7 @@ func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string, a
storages := t.getStorages(orgId)
count := len(storages)
grafanaStorageLogger.Info("Listing root folder", "path", path, "storageCount", len(storages))
names := data.NewFieldFromFieldType(data.FieldTypeString, count)
title := data.NewFieldFromFieldType(data.FieldTypeString, count)
@ -154,23 +203,38 @@ func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string, a
return nil, nil // not found (or not ready)
}
var storages []storageRuntime
if root.Meta().Config.Prefix == RootContent && (path == "" || path == "/") {
storages = filterStoragesUnderContentRoot(t.getStorages(orgId))
}
grafanaStorageLogger.Info("Listing folder", "path", path, "storageCount", len(storages), "root", root.Meta().Config.Prefix)
store := root.Store()
if store == nil {
return nil, fmt.Errorf("store not ready")
}
pathFilter := accessFilter
if root.Meta().Config.Prefix == RootContent && len(storages) > 0 {
// create a PathFilter that will filter out folders that are "shadowed" by the mounted storages
pathFilter = filestorage.NewAndPathFilter(
accessFilter,
t.createPathFilterForContentRoot(storages),
)
}
listResponse, err := store.List(ctx, path, nil, &filestorage.ListOptions{
Recursive: false,
WithFolders: true,
WithFiles: true,
Filter: accessFilter,
Filter: pathFilter,
})
if err != nil {
return nil, err
}
count := len(listResponse.Files)
count := len(listResponse.Files) + len(storages)
names := data.NewFieldFromFieldType(data.FieldTypeString, count)
mtype := data.NewFieldFromFieldType(data.FieldTypeString, count)
fsize := data.NewFieldFromFieldType(data.FieldTypeInt64, count)
@ -180,11 +244,22 @@ func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string, a
fsize.Config = &data.FieldConfig{
Unit: "bytes",
}
for i, f := range listResponse.Files {
names.Set(i, f.Name)
mtype.Set(i, f.MimeType)
fsize.Set(i, f.Size)
idx := 0
for _, s := range storages {
names.Set(idx, s.Meta().Config.Prefix)
mtype.Set(idx, filestorage.DirectoryMimeType)
fsize.Set(idx, int64(0))
idx++
}
for _, f := range listResponse.Files {
names.Set(idx, f.Name)
mtype.Set(idx, f.MimeType)
fsize.Set(idx, f.Size)
idx++
}
frame := data.NewFrame("", names, mtype, fsize)
frame.SetMeta(&data.FrameMeta{
Type: data.FrameTypeDirectoryListing,
@ -194,3 +269,17 @@ func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string, a
})
return &StorageListFrame{frame}, nil
}
func (t *nestedTree) createPathFilterForContentRoot(storages []storageRuntime) filestorage.PathFilter {
disallowedPrefixes := make([]string, 0)
disallowedPaths := make([]string, 0)
for _, s := range storages {
path := filestorage.Delimiter + s.Meta().Config.Prefix
disallowedPaths = append(disallowedPaths, path)
disallowedPrefixes = append(disallowedPrefixes, path+filestorage.Delimiter)
}
grafanaStorageLogger.Info("Created a path filter for the content root", "disallowedPrefixes", disallowedPrefixes, "disallowedPaths", disallowedPaths)
return filestorage.NewPathFilter(nil, nil, disallowedPrefixes, disallowedPaths)
}

62
public/app/features/storage/RootView.tsx
View File

@ -35,10 +35,10 @@ export function RootView({ root, onPathChange }: Props) {
}
const roots = useMemo(() => {
const all = storage.value;
if (searchQuery?.length && all) {
let show = storage.value ?? [];
if (searchQuery?.length) {
const lower = searchQuery.toLowerCase();
return all.filter((r) => {
show = show.filter((r) => {
const v = r.config;
const isMatch = v.name.toLowerCase().indexOf(lower) >= 0 || v.description.toLowerCase().indexOf(lower) >= 0;
if (isMatch) {
@ -47,29 +47,26 @@ export function RootView({ root, onPathChange }: Props) {
return false;
});
}
return all ?? [];
const base: StorageInfo[] = [];
const content: StorageInfo[] = [];
for (const r of show ?? []) {
if (r.config.underContentRoot) {
content.push(r);
} else if (r.config.prefix !== 'content') {
base.push(r);
}
}
return { base, content };
}, [searchQuery, storage]);
return (
<div>
<div className="page-action-bar">
<div className="gf-form gf-form--grow">
<FilterInput placeholder="Search Storage" value={searchQuery} onChange={setSearchQuery} />
</div>
<Button className="pull-right" onClick={() => onPathChange('', StorageView.AddRoot)}>
Add Root
</Button>
{config.featureToggles.export && (
<Button className="pull-right" onClick={() => onPathChange('', StorageView.Export)}>
Export
</Button>
)}
</div>
const renderRoots = (pfix: string, roots: StorageInfo[]) => {
return (
<VerticalGroup>
{roots.map((s) => {
const ok = s.ready;
return (
<Card key={s.config.prefix} href={ok ? `admin/storage/${s.config.prefix}/` : undefined}>
<Card key={s.config.prefix} href={ok ? `admin/storage/${pfix}${s.config.prefix}/` : undefined}>
<Card.Heading>{s.config.name}</Card.Heading>
<Card.Meta className={styles.clickable}>
{s.config.description}
@ -91,6 +88,31 @@ export function RootView({ root, onPathChange }: Props) {
);
})}
</VerticalGroup>
);
};
return (
<div>
<div className="page-action-bar">
<div className="gf-form gf-form--grow">
<FilterInput placeholder="Search Storage" value={searchQuery} onChange={setSearchQuery} />
</div>
<Button className="pull-right" onClick={() => onPathChange('', StorageView.AddRoot)}>
Add Root
</Button>
{config.featureToggles.export && (
<Button className="pull-right" onClick={() => onPathChange('', StorageView.Export)}>
Export
</Button>
)}
</div>
<div>{renderRoots('', roots.base)}</div>
<div>
<h3>Content</h3>
{renderRoots('content/', roots.content)}
</div>
</div>
);
}

2
public/app/features/storage/StorageFolderPage.tsx
View File

@ -14,7 +14,7 @@ export interface Props extends GrafanaRouteComponentProps<{ slug: string }> {}
export function StorageFolderPage(props: Props) {
const slug = props.match.params.slug ?? '';
const listing = useAsync((): Promise<DataFrame | undefined> => {
return getGrafanaStorage().list(slug);
return getGrafanaStorage().list('content/' + slug);
}, [slug]);
const childRoot = slug.length > 0 ? `g/${slug}/` : 'g/';

6
public/app/features/storage/StoragePage.tsx
View File

@ -155,9 +155,9 @@ export default function StoragePage(props: Props) {
opts.push({ what: StorageView.History, text: 'History' });
}
const canAddFolder = isFolder && path.startsWith('resources');
const canDelete = path.startsWith('resources/');
const canViewDashboard = config.featureToggles.dashboardsFromStorage && (isFolder || path.endsWith('.json'));
const canAddFolder = isFolder && (path.startsWith('resources') || path.startsWith('content'));
const canDelete = path.startsWith('resources/') || path.startsWith('content/');
const canViewDashboard = config.featureToggles.dashboardsFromStorage && path.startsWith('content/');
const getErrorMessages = () => {
return (

4
public/app/features/storage/storage.ts
View File

@ -131,6 +131,10 @@ class SimpleStorage implements GrafanaStorage {
path += '.json';
}
if (!path.startsWith('content/')) {
path = `content/${path}`;
}
const result = await backendSrv.get(`/api/storage/read/${path}`);
result.uid = path;
delete result.id; // Saved with the dev dashboards!

1
public/app/features/storage/types.ts
View File

@ -31,6 +31,7 @@ export interface StorageConfig {
prefix: string;
name: string;
description: string;
underContentRoot: string;
disk?: {
path: string;
};