grafana/pkg/services/store/service_test.go
Dan Cech 790e1feb93
Chore: Update test database initialization (#81673)
* streamline initialization of test databases, support on-disk sqlite test db

* clean up test databases

* introduce testsuite helper

* use testsuite everywhere we use a test db

* update documentation

* improve error handling

* disable entity integration test until we can figure out locking error
2024-02-09 09:35:39 -05:00

560 lines
19 KiB
Go

package store
import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/grafana/grafana-plugin-sdk-go/experimental"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/filestorage"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/quota/quotatest"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tests/testsuite"
testdatasource "github.com/grafana/grafana/pkg/tsdb/grafana-testdata-datasource"
)
var (
cfg = &setting.Cfg{
Storage: setting.StorageSettings{
AllowUnsanitizedSvgUpload: true,
},
}
htmlBytes, _ = os.ReadFile("testdata/page.html")
jpgBytes, _ = os.ReadFile("testdata/image.jpg")
svgBytes, _ = os.ReadFile("testdata/image.svg")
dummyUser = &user.SignedInUser{OrgID: 1}
globalUser = &user.SignedInUser{OrgID: 0}
allowAllAuthService = newStaticStorageAuthService(func(ctx context.Context, user *user.SignedInUser, storageName string) map[string]filestorage.PathFilter {
return map[string]filestorage.PathFilter{
ActionFilesDelete: allowAllPathFilter,
ActionFilesWrite: allowAllPathFilter,
ActionFilesRead: allowAllPathFilter,
}
})
denyAllAuthService = newStaticStorageAuthService(func(ctx context.Context, user *user.SignedInUser, storageName string) map[string]filestorage.PathFilter {
return map[string]filestorage.PathFilter{
ActionFilesDelete: denyAllPathFilter,
ActionFilesWrite: denyAllPathFilter,
ActionFilesRead: denyAllPathFilter,
}
})
publicRoot, _ = filepath.Abs("../../../public")
publicStaticFilesStorage = newDiskStorage(
RootStorageMeta{
Builtin: true,
ReadOnly: true,
}, RootStorageConfig{
Prefix: "public",
Name: "Public static files",
Disk: &StorageLocalDiskConfig{
Path: publicRoot,
Roots: []string{
"/img/icons/",
"/img/bg/",
"/gazetteer/",
"/maps/",
"/upload/",
},
}})
)
func TestMain(m *testing.M) {
testsuite.Run(m)
}
func TestListFiles(t *testing.T) {
roots := []storageRuntime{publicStaticFilesStorage}
store := newStandardStorageService(db.InitTestDB(t), roots, func(orgId int64) []storageRuntime {
return make([]storageRuntime, 0)
}, allowAllAuthService, cfg, nil)
frame, err := store.List(context.Background(), dummyUser, "public/maps", 0)
require.NoError(t, err)
experimental.CheckGoldenJSONFrame(t, "testdata", "public_testdata.golden", frame.Frame, true)
file, err := store.Read(context.Background(), dummyUser, "public/maps/countries.geojson")
require.NoError(t, err)
require.NotNil(t, file)
t.Skip("Skipping golden JSON frame test as it is flaky")
testDsFrame, err := testdatasource.LoadCsvContent(bytes.NewReader(file.Contents), file.Name)
require.NoError(t, err)
experimental.CheckGoldenJSONFrame(t, "testdata", "public_testdata_js_libraries.golden", testDsFrame, true)
}
func TestListFilesWithoutPermissions(t *testing.T) {
roots := []storageRuntime{publicStaticFilesStorage}
store := newStandardStorageService(db.InitTestDB(t), roots, func(orgId int64) []storageRuntime {
return make([]storageRuntime, 0)
}, denyAllAuthService, cfg, nil)
frame, err := store.List(context.Background(), dummyUser, "public/maps", 0)
require.NoError(t, err)
rowLen, err := frame.RowLen()
require.NoError(t, err)
require.Equal(t, 0, rowLen)
}
func setupUploadStore(t *testing.T, authService storageAuthService) (StorageService, *filestorage.MockFileStorage, string) {
t.Helper()
storageName := "resources"
mockStorage := &filestorage.MockFileStorage{}
sqlStorage := newSQLStorage(RootStorageMeta{}, storageName, "Testing upload", "dummy descr", &StorageSQLConfig{}, db.InitTestDB(t), 1, false)
sqlStorage.store = mockStorage
if authService == nil {
authService = allowAllAuthService
}
store := newStandardStorageService(db.InitTestDB(t), []storageRuntime{sqlStorage}, func(orgId int64) []storageRuntime {
return make([]storageRuntime, 0)
}, authService, cfg, nil)
store.cfg = &GlobalStorageConfig{
AllowUnsanitizedSvgUpload: true,
}
store.quotaService = quotatest.New(false, nil)
return store, mockStorage, storageName
}
func TestShouldUploadWhenNoFileAlreadyExists(t *testing.T) {
service, mockStorage, storageName := setupUploadStore(t, nil)
fileName := "/myFile.jpg"
mockStorage.On("Get", mock.Anything, fileName, &filestorage.GetFileOptions{WithContents: false}).Return(nil, false, nil)
mockStorage.On("Upsert", mock.Anything, &filestorage.UpsertFileCommand{
Path: fileName,
MimeType: "image/jpeg",
Contents: jpgBytes,
}).Return(nil)
err := service.Upload(context.Background(), dummyUser, &UploadRequest{
EntityType: EntityTypeImage,
Contents: jpgBytes,
Path: storageName + fileName,
})
require.NoError(t, err)
}
func TestShouldFailUploadWithoutAccess(t *testing.T) {
service, _, storageName := setupUploadStore(t, denyAllAuthService)
err := service.Upload(context.Background(), dummyUser, &UploadRequest{
EntityType: EntityTypeImage,
Contents: jpgBytes,
Path: storageName + "/myFile.jpg",
})
require.ErrorIs(t, err, ErrAccessDenied)
}
func TestShouldFailUploadWhenFileAlreadyExists(t *testing.T) {
service, mockStorage, storageName := setupUploadStore(t, nil)
mockStorage.On("Get", mock.Anything, "/myFile.jpg", &filestorage.GetFileOptions{WithContents: false}).Return(&filestorage.File{Contents: make([]byte, 0)}, true, nil)
err := service.Upload(context.Background(), dummyUser, &UploadRequest{
EntityType: EntityTypeImage,
Contents: jpgBytes,
Path: storageName + "/myFile.jpg",
})
require.ErrorIs(t, err, ErrFileAlreadyExists)
}
func TestShouldDelegateFileDeletion(t *testing.T) {
service, mockStorage, storageName := setupUploadStore(t, nil)
mockStorage.On("Delete", mock.Anything, "/myFile.jpg").Return(nil)
err := service.Delete(context.Background(), dummyUser, storageName+"/myFile.jpg")
require.NoError(t, err)
}
func TestShouldDelegateFolderCreation(t *testing.T) {
service, mockStorage, storageName := setupUploadStore(t, nil)
mockStorage.On("CreateFolder", mock.Anything, "/nestedFolder/mostNestedFolder").Return(nil)
err := service.CreateFolder(context.Background(), dummyUser, &CreateFolderCmd{Path: storageName + "/nestedFolder/mostNestedFolder"})
require.NoError(t, err)
}
func TestShouldDelegateFolderDeletion(t *testing.T) {
service, mockStorage, storageName := setupUploadStore(t, nil)
cmds := []*DeleteFolderCmd{
{
Path: storageName,
Force: false,
},
{
Path: storageName,
Force: true,
}}
ctx := context.Background()
for _, cmd := range cmds {
mockStorage.On("DeleteFolder", ctx, "/", &filestorage.DeleteFolderOptions{
Force: cmd.Force,
AccessFilter: allowAllPathFilter,
}).Once().Return(nil)
err := service.DeleteFolder(ctx, dummyUser, cmd)
require.NoError(t, err)
}
}
func TestShouldUploadSvg(t *testing.T) {
service, mockStorage, storageName := setupUploadStore(t, nil)
fileName := "/myFile.svg"
mockStorage.On("Get", mock.Anything, fileName, &filestorage.GetFileOptions{WithContents: false}).Return(nil, false, nil)
mockStorage.On("Upsert", mock.Anything, &filestorage.UpsertFileCommand{
Path: fileName,
MimeType: "image/svg+xml",
Contents: svgBytes,
}).Return(nil)
err := service.Upload(context.Background(), dummyUser, &UploadRequest{
EntityType: EntityTypeImage,
Contents: svgBytes,
Path: storageName + fileName,
})
require.NoError(t, err)
}
func TestShouldNotUploadHtmlDisguisedAsSvg(t *testing.T) {
service, mockStorage, storageName := setupUploadStore(t, nil)
fileName := "/myFile.svg"
mockStorage.On("Get", mock.Anything, fileName).Return(nil, nil)
err := service.Upload(context.Background(), dummyUser, &UploadRequest{
EntityType: EntityTypeImage,
Contents: htmlBytes,
Path: storageName + fileName,
})
require.ErrorIs(t, err, ErrValidationFailed)
}
func TestShouldNotUploadJpgDisguisedAsSvg(t *testing.T) {
service, mockStorage, storageName := setupUploadStore(t, nil)
fileName := "/myFile.svg"
mockStorage.On("Get", mock.Anything, fileName).Return(nil, nil)
err := service.Upload(context.Background(), dummyUser, &UploadRequest{
EntityType: EntityTypeImage,
Contents: jpgBytes,
Path: storageName + fileName,
})
require.ErrorIs(t, err, ErrValidationFailed)
}
func TestSetupWithNonUniqueStoragePrefixes(t *testing.T) {
prefix := "resources"
sqlStorage := newSQLStorage(RootStorageMeta{}, prefix, "Testing upload", "dummy descr", &StorageSQLConfig{}, db.InitTestDB(t), 1, false)
sqlStorage2 := newSQLStorage(RootStorageMeta{}, prefix, "Testing upload", "dummy descr", &StorageSQLConfig{}, db.InitTestDB(t), 1, false)
defer func() {
if r := recover(); r == nil {
t.Errorf("The setup should have panicked")
}
}()
newStandardStorageService(db.InitTestDB(t), []storageRuntime{sqlStorage, sqlStorage2}, func(orgId int64) []storageRuntime {
return make([]storageRuntime, 0)
}, allowAllAuthService, cfg, nil)
}
func TestContentRootWithNestedStorage(t *testing.T) {
globalOrgID := int64(accesscontrol.GlobalOrgID)
testDB := db.InitTestDB(t)
orgedUser := &user.SignedInUser{OrgID: 1}
t.Helper()
mockContentFSApi := &filestorage.MockFileStorage{}
contentStorage := newSQLStorage(RootStorageMeta{}, RootContent, "Content root", "dummy descr", &StorageSQLConfig{}, testDB, globalOrgID, false)
contentStorage.store = mockContentFSApi
nestedRoot := "nested"
mockNestedFSApi := &filestorage.MockFileStorage{}
nestedStorage := newSQLStorage(RootStorageMeta{}, nestedRoot, "Nested root", "dummy descr", &StorageSQLConfig{}, testDB, globalOrgID, true)
nestedStorage.store = mockNestedFSApi
nestedOrgedRoot := "nestedOrged"
mockNestedOrgedFSApi := &filestorage.MockFileStorage{}
nestedOrgedStorage := newSQLStorage(RootStorageMeta{}, nestedOrgedRoot, "Nested root", "dummy descr", &StorageSQLConfig{}, testDB, globalOrgID, true)
nestedOrgedStorage.store = mockNestedOrgedFSApi
store := newStandardStorageService(db.InitTestDB(t), []storageRuntime{contentStorage, nestedStorage}, func(orgId int64) []storageRuntime {
return []storageRuntime{nestedOrgedStorage, contentStorage}
}, allowAllAuthService, cfg, nil)
store.cfg = &GlobalStorageConfig{
AllowUnsanitizedSvgUpload: true,
}
store.quotaService = quotatest.New(false, nil)
fileName := "file.jpg"
tests := []struct {
user *user.SignedInUser
name string
mockNestedFS *filestorage.MockFileStorage
nestedRoot string
}{
{
user: globalUser,
name: "global user, global nested storage",
mockNestedFS: mockNestedFSApi,
nestedRoot: nestedRoot,
},
{
user: orgedUser,
name: "non-global user, global nested storage",
mockNestedFS: mockNestedFSApi,
nestedRoot: nestedRoot,
},
{
user: orgedUser,
name: "non-global user, non-global nested storage",
mockNestedFS: mockNestedOrgedFSApi,
nestedRoot: nestedOrgedRoot,
},
}
for _, test := range tests {
t.Run(test.name+": Uploading a file under a /content/nested/.. should delegate to the nested storage", func(t *testing.T) {
test.mockNestedFS.On("Get", mock.Anything, filestorage.Delimiter+fileName, &filestorage.GetFileOptions{WithContents: false}).Return(nil, false, nil)
test.mockNestedFS.On("Upsert", mock.Anything, &filestorage.UpsertFileCommand{
Path: filestorage.Delimiter + fileName,
MimeType: "image/jpeg",
Contents: jpgBytes,
}).Return(nil)
mockContentFSApi.AssertNotCalled(t, "Get")
mockContentFSApi.AssertNotCalled(t, "Upsert")
err := store.Upload(context.Background(), test.user, &UploadRequest{
EntityType: EntityTypeImage,
Contents: jpgBytes,
Path: strings.Join([]string{RootContent, test.nestedRoot, fileName}, filestorage.Delimiter),
})
require.NoError(t, err)
})
t.Run(test.name+": Creating a /content/nested folder should fail", func(t *testing.T) {
mockContentFSApi.AssertNotCalled(t, "CreateFolder")
err := store.CreateFolder(context.Background(), test.user, &CreateFolderCmd{Path: RootContent + "/" + test.nestedRoot})
require.ErrorIs(t, err, ErrValidationFailed)
})
t.Run(test.name+": Deleting a /content/nested folder should fail", func(t *testing.T) {
mockContentFSApi.AssertNotCalled(t, "DeleteFolder")
err := store.DeleteFolder(context.Background(), test.user, &DeleteFolderCmd{Path: RootContent + "/" + test.nestedRoot})
require.ErrorIs(t, err, ErrValidationFailed)
})
t.Run(test.name+": Listing /content/nested should delegate to the nested root", func(t *testing.T) {
mockContentFSApi.AssertNotCalled(t, "List")
test.mockNestedFS.On(
"List",
mock.Anything,
"/",
mock.Anything,
mock.Anything,
).Return(&filestorage.ListResponse{
Files: []*filestorage.File{},
}, nil)
_, err := store.List(context.Background(), test.user, RootContent+"/"+test.nestedRoot, 0)
require.NoError(t, err)
})
t.Run(test.name+": Listing a folder inside /content/nested/.. should delegate to the nested root", func(t *testing.T) {
mockContentFSApi.AssertNotCalled(t, "List")
test.mockNestedFS.On(
"List",
mock.Anything,
"/folder1/folder2",
mock.Anything,
mock.Anything,
).Return(&filestorage.ListResponse{
Files: []*filestorage.File{},
}, nil)
_, err := store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot, "folder1", "folder2"}, "/"), 0)
require.NoError(t, err)
})
t.Run(test.name+": Listing outside of the nested storages should delegate to the content root", func(t *testing.T) {
test.mockNestedFS.AssertNotCalled(t, "List")
mockContentFSApi.On(
"List",
mock.Anything,
"/not-nested-content",
mock.Anything,
mock.Anything,
).Return(&filestorage.ListResponse{
Files: []*filestorage.File{},
}, nil)
mockContentFSApi.On(
"List",
mock.Anything,
"/a/b/c",
mock.Anything,
mock.Anything,
).Return(&filestorage.ListResponse{
Files: []*filestorage.File{},
}, nil)
mockContentFSApi.On(
"List",
mock.Anything,
fmt.Sprintf("/%sa", test.nestedRoot),
mock.Anything,
mock.Anything,
).Return(&filestorage.ListResponse{
Files: []*filestorage.File{},
}, nil)
mockContentFSApi.On(
"List",
mock.Anything,
fmt.Sprintf("/%sa/b", test.nestedRoot),
mock.Anything,
mock.Anything,
).Return(&filestorage.ListResponse{
Files: []*filestorage.File{},
}, nil)
_, err := store.List(context.Background(), test.user, strings.Join([]string{RootContent, "not-nested-content"}, "/"), 0)
require.NoError(t, err)
_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, "a", "b", "c"}, "/"), 0)
require.NoError(t, err)
_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot + "a"}, "/"), 0)
require.NoError(t, err)
_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot + "a", "b"}, "/"), 0)
require.NoError(t, err)
})
t.Run(test.name+": Uploading files outside of the nested storages should delegate to the content root", func(t *testing.T) {
test.mockNestedFS.AssertNotCalled(t, "Get")
test.mockNestedFS.AssertNotCalled(t, "Upsert")
// file at the root of the content root - /content/myFile.jpg
fileName := "myFile.jpg"
mockContentFSApi.On("Get", mock.Anything, "/"+fileName, &filestorage.GetFileOptions{WithContents: false}).Return(nil, false, nil)
mockContentFSApi.On("Upsert", mock.Anything, &filestorage.UpsertFileCommand{
Path: "/" + fileName,
MimeType: "image/jpeg",
Contents: jpgBytes,
}).Return(nil)
err := store.Upload(context.Background(), dummyUser, &UploadRequest{
EntityType: EntityTypeImage,
Contents: jpgBytes,
Path: strings.Join([]string{RootContent, fileName}, "/"),
})
require.NoError(t, err)
// file in the folder belonging to the content root storage - /content/nested/a/myFile.jpg
mockContentFSApi.On("Get", mock.Anything, "/a/"+fileName, &filestorage.GetFileOptions{WithContents: false}).Return(nil, false, nil)
mockContentFSApi.On("Upsert", mock.Anything, &filestorage.UpsertFileCommand{
Path: "/a/" + fileName,
MimeType: "image/jpeg",
Contents: jpgBytes,
}).Return(nil)
err = store.Upload(context.Background(), dummyUser, &UploadRequest{
EntityType: EntityTypeImage,
Contents: jpgBytes,
Path: strings.Join([]string{RootContent, "a", fileName}, "/"),
})
require.NoError(t, err)
})
t.Run(test.name+": Creating folders under /content/nested/.. should delegate to the nested roots", func(t *testing.T) {
mockContentFSApi.AssertNotCalled(t, "CreateFolder")
mockContentFSApi.AssertNotCalled(t, "DeleteFolder")
test.mockNestedFS.On("CreateFolder", mock.Anything, "/folder").Return(nil)
path := strings.Join([]string{RootContent, test.nestedRoot, "folder"}, "/")
err := store.CreateFolder(context.Background(), test.user, &CreateFolderCmd{Path: path})
require.NoError(t, err)
test.mockNestedFS.On("DeleteFolder", mock.Anything, "/folder", mock.Anything).Return(nil)
err = store.DeleteFolder(context.Background(), test.user, &DeleteFolderCmd{Path: path})
require.NoError(t, err)
})
t.Run(test.name+": Creating folders under outside of the nested storages should delegate to the content root", func(t *testing.T) {
test.mockNestedFS.AssertNotCalled(t, "CreateFolder")
test.mockNestedFS.AssertNotCalled(t, "DeleteFolder")
mockContentFSApi.On("CreateFolder", mock.Anything, "/folder").Return(nil)
path := strings.Join([]string{RootContent, "folder"}, "/")
err := store.CreateFolder(context.Background(), test.user, &CreateFolderCmd{Path: path})
require.NoError(t, err)
mockContentFSApi.On("DeleteFolder", mock.Anything, "/folder", mock.Anything).Return(nil)
err = store.DeleteFolder(context.Background(), test.user, &DeleteFolderCmd{Path: path})
require.NoError(t, err)
})
}
}
func TestShadowingExistingFolderByNestedContentRoot(t *testing.T) {
db := db.InitTestDB(t)
ctx := context.Background()
nestedStorage := newSQLStorage(RootStorageMeta{}, "nested", "Testing upload", "dummy descr", &StorageSQLConfig{}, db, accesscontrol.GlobalOrgID, true)
contentStorage := newSQLStorage(RootStorageMeta{}, RootContent, "Testing upload", "dummy descr", &StorageSQLConfig{}, db, accesscontrol.GlobalOrgID, false)
_, err := contentStorage.Write(ctx, &WriteValueRequest{
User: globalUser,
Path: "/nested/abc.jpg",
EntityType: EntityTypeImage,
Body: jpgBytes,
})
require.NoError(t, err)
store := newStandardStorageService(db, []storageRuntime{nestedStorage, contentStorage}, func(orgId int64) []storageRuntime { return make([]storageRuntime, 0) }, allowAllAuthService, cfg, nil)
store.cfg = &GlobalStorageConfig{
AllowUnsanitizedSvgUpload: true,
}
resp, err := store.List(ctx, globalUser, "content/nested", 0)
require.NoError(t, err)
require.NotNil(t, resp)
rowLen, err := resp.Frame.RowLen()
require.NoError(t, err)
require.Equal(t, 0, rowLen) // nested storage is empty
resp, err = store.List(ctx, globalUser, "content", 0)
require.NoError(t, err)
require.NotNil(t, resp)
rowLen, err = resp.Frame.RowLen()
require.NoError(t, err)
require.Equal(t, 1, rowLen) // just a single "nested" folder
}