IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Storage: Add maxFiles to list functions (#76414)

Browse Source 
* Storage: Add maxFiles to list functions

* Add maxDataPoints argument to listFiles function

* Add maxFiles to ResourceDimensionEditor

* Update pkg/services/store/http.go

* rename First to Limit

---------

Co-authored-by: jennyfana <110450222+jennyfana@users.noreply.github.com>
Co-authored-by: nmarrs <nathanielmarrs@gmail.com>
Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
This commit is contained in:
Drew Slobodnjak 2023-11-01 09:42:24 -07:00 committed by GitHub
parent f6d3238505
commit 9116043453
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 78 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/infra/filestorage/api.go
View File

@ -93,8 +93,10 @@ type FileMetadata struct {
}
type Paging struct {
// The number of items to return
Limit int
// Starting after the key
After string
First int
}
type UpsertFileCommand struct {

2
pkg/infra/filestorage/cdk_blob_filestorage.go
View File

@ -292,7 +292,7 @@ func (c cdkBlobStorage) list(ctx context.Context, folderPath string, paging *Pag
})}
recursive := options.Recursive
pageSize := paging.First
pageSize := paging.Limit
foundCursor := true
if paging.After != "" {

2
pkg/infra/filestorage/db_filestorage.go
View File

@ -345,7 +345,7 @@ func (s dbFileStorage) List(ctx context.Context, folderPath string, paging *Pagi
sess.OrderBy("path")
pageSize := paging.First
pageSize := paging.Limit
sess.Limit(pageSize + 1)
if cursor != "" {

28
pkg/infra/filestorage/fs_integration_test.go
View File

@ -467,7 +467,7 @@ func TestIntegrationFsStorage(t *testing.T) {
},
},
queryListFiles{
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 2, After: ""}},
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{Limit: 2, After: ""}},
list: checks(listSize(2), listHasMore(true), listLastPath("/folder1/b")),
files: [][]any{
checks(fPath("/folder1/a")),
@ -475,7 +475,7 @@ func TestIntegrationFsStorage(t *testing.T) {
},
},
queryListFiles{
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 2, After: ""}},
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{Limit: 2, After: ""}},
list: checks(listSize(2), listHasMore(true), listLastPath("/folder1/a")),
files: [][]any{
checks(fPath("/folder1"), fMimeType(DirectoryMimeType)),
@ -483,49 +483,49 @@ func TestIntegrationFsStorage(t *testing.T) {
},
},
queryListFiles{
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: "/folder1"}},
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{Limit: 1, After: "/folder1"}},
list: checks(listSize(1), listHasMore(true), listLastPath("/folder1/a")),
files: [][]any{
checks(fPath("/folder1/a")),
},
},
queryListFiles{
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: "/folder1/a"}},
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{Limit: 1, After: "/folder1/a"}},
list: checks(listSize(1), listHasMore(true), listLastPath("/folder1/b")),
files: [][]any{
checks(fPath("/folder1/b")),
},
},
queryListFiles{
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: "/folder1/a"}},
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{Limit: 1, After: "/folder1/a"}},
list: checks(listSize(1), listHasMore(true), listLastPath("/folder1/b")),
files: [][]any{
checks(fPath("/folder1/b")),
},
},
queryListFiles{
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: "/folder1/b"}},
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{Limit: 1, After: "/folder1/b"}},
list: checks(listSize(1), listHasMore(false), listLastPath("/folder2/c")),
files: [][]any{
checks(fPath("/folder2/c")),
},
},
queryListFiles{
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: "/folder1/b"}},
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{Limit: 1, After: "/folder1/b"}},
list: checks(listSize(1), listHasMore(true), listLastPath("/folder2")),
files: [][]any{
checks(fPath("/folder2"), fMimeType(DirectoryMimeType)),
},
},
queryListFiles{
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: "/folder2"}},
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{Limit: 1, After: "/folder2"}},
list: checks(listSize(1), listHasMore(false), listLastPath("/folder2/c")),
files: [][]any{
checks(fPath("/folder2/c")),
},
},
queryListFiles{
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: ""}},
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{Limit: 5, After: ""}},
list: checks(listSize(3), listHasMore(false), listLastPath("/folder2/c")),
files: [][]any{
checks(fPath("/folder1/a")),
@ -534,7 +534,7 @@ func TestIntegrationFsStorage(t *testing.T) {
},
},
queryListFiles{
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 5, After: ""}},
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{Limit: 5, After: ""}},
list: checks(listSize(5), listHasMore(false), listLastPath("/folder2/c")),
files: [][]any{
checks(fPath("/folder1"), fMimeType(DirectoryMimeType)),
@ -545,19 +545,19 @@ func TestIntegrationFsStorage(t *testing.T) {
},
},
queryListFiles{
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: "/folder2"}},
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{Limit: 5, After: "/folder2"}},
list: checks(listSize(1), listHasMore(false)),
},
queryListFiles{
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 5, After: "/folder2"}},
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{Limit: 5, After: "/folder2"}},
list: checks(listSize(1), listHasMore(false)),
},
queryListFiles{
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: "/folder2/c"}},
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{Limit: 5, After: "/folder2/c"}},
list: checks(listSize(0), listHasMore(false)),
},
queryListFiles{
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 5, After: "/folder2/c"}},
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{Limit: 5, After: "/folder2/c"}},
list: checks(listSize(0), listHasMore(false)),
},
},

2
pkg/infra/filestorage/test_utils.go
View File

@ -321,7 +321,7 @@ func handleQuery(t *testing.T, ctx context.Context, query interface{}, queryName
}
resp, err := fs.List(ctx, inputPath, &Paging{
After: "",
First: 100000,
Limit: 100000,
}, opts)
require.NotNil(t, resp)
require.NoError(t, err, "%s: should be able to list folders in %s", queryName, inputPath)

8
pkg/infra/filestorage/wrapper.go
View File

@ -200,12 +200,12 @@ func (b wrapper) Upsert(ctx context.Context, file *UpsertFileCommand) error {
func (b wrapper) pagingOptionsWithDefaults(paging *Paging) *Paging {
if paging == nil {
return &Paging{
First: 100,
Limit: 100,
}
}
if paging.First <= 0 {
paging.First = 100
if paging.Limit <= 0 {
paging.Limit = 100
}
if paging.After != "" {
paging.After = b.addRoot(paging.After)
@ -381,7 +381,7 @@ func (b wrapper) List(ctx context.Context, folderPath string, paging *Paging, op
}
func (b wrapper) isFolderEmpty(ctx context.Context, path string) (bool, error) {
resp, err := b.List(ctx, path, &Paging{First: 1}, &ListOptions{Recursive: true, WithFolders: true, WithFiles: true})
resp, err := b.List(ctx, path, &Paging{Limit: 1}, &ListOptions{Recursive: true, WithFolders: true, WithFiles: true})
if err != nil {
return false, err
}

3
pkg/services/store/http.go
View File

@ -260,7 +260,8 @@ func (s *standardStorageService) doCreateFolder(c *contextmodel.ReqContext) resp
func (s *standardStorageService) list(c *contextmodel.ReqContext) response.Response {
params := web.Params(c.Req)
path := params["*"]
frame, err := s.List(c.Req.Context(), c.SignedInUser, path)
// maxFiles of 0 will result in default behaviour from wrapper
frame, err := s.List(c.Req.Context(), c.SignedInUser, path, 0)
if err != nil {
return response.Error(400, "error reading path", err)
}

6
pkg/services/store/service.go
View File

@ -64,7 +64,7 @@ type StorageService interface {
RegisterHTTPRoutes(routing.RouteRegister)
// List folder contents
List(ctx context.Context, user *user.SignedInUser, path string) (*StorageListFrame, error)
List(ctx context.Context, user *user.SignedInUser, path string, maxFiles int) (*StorageListFrame, error)
// Read raw file contents out of the store
Read(ctx context.Context, user *user.SignedInUser, path string) (*filestorage.File, error)
@ -340,9 +340,9 @@ func getOrgId(user *user.SignedInUser) int64 {
return user.OrgID
}
func (s *standardStorageService) List(ctx context.Context, user *user.SignedInUser, path string) (*StorageListFrame, error) {
func (s *standardStorageService) List(ctx context.Context, user *user.SignedInUser, path string, maxFiles int) (*StorageListFrame, error) {
guardian := s.authService.newGuardian(ctx, user, getFirstSegment(path))
return s.tree.ListFolder(ctx, getOrgId(user), path, guardian.getPathFilter(ActionFilesRead))
return s.tree.ListFolder(ctx, getOrgId(user), path, maxFiles, guardian.getPathFilter(ActionFilesRead))
}
func (s *standardStorageService) Read(ctx context.Context, user *user.SignedInUser, path string) (*filestorage.File, error) {

20
pkg/services/store/service_test.go
View File

@ -74,7 +74,7 @@ func TestListFiles(t *testing.T) {
store := newStandardStorageService(db.InitTestDB(t), roots, func(orgId int64) []storageRuntime {
return make([]storageRuntime, 0)
}, allowAllAuthService, cfg, nil)
frame, err := store.List(context.Background(), dummyUser, "public/maps")
frame, err := store.List(context.Background(), dummyUser, "public/maps", 0)
require.NoError(t, err)
experimental.CheckGoldenJSONFrame(t, "testdata", "public_testdata.golden", frame.Frame, true)
@ -95,7 +95,7 @@ func TestListFilesWithoutPermissions(t *testing.T) {
store := newStandardStorageService(db.InitTestDB(t), roots, func(orgId int64) []storageRuntime {
return make([]storageRuntime, 0)
}, denyAllAuthService, cfg, nil)
frame, err := store.List(context.Background(), dummyUser, "public/maps")
frame, err := store.List(context.Background(), dummyUser, "public/maps", 0)
require.NoError(t, err)
rowLen, err := frame.RowLen()
require.NoError(t, err)
@ -371,7 +371,7 @@ func TestContentRootWithNestedStorage(t *testing.T) {
Files: []*filestorage.File{},
}, nil)
_, err := store.List(context.Background(), test.user, RootContent+"/"+test.nestedRoot)
_, err := store.List(context.Background(), test.user, RootContent+"/"+test.nestedRoot, 0)
require.NoError(t, err)
})
@ -387,7 +387,7 @@ func TestContentRootWithNestedStorage(t *testing.T) {
Files: []*filestorage.File{},
}, nil)
_, err := store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot, "folder1", "folder2"}, "/"))
_, err := store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot, "folder1", "folder2"}, "/"), 0)
require.NoError(t, err)
})
@ -434,16 +434,16 @@ func TestContentRootWithNestedStorage(t *testing.T) {
Files: []*filestorage.File{},
}, nil)
_, err := store.List(context.Background(), test.user, strings.Join([]string{RootContent, "not-nested-content"}, "/"))
_, err := store.List(context.Background(), test.user, strings.Join([]string{RootContent, "not-nested-content"}, "/"), 0)
require.NoError(t, err)
_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, "a", "b", "c"}, "/"))
_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, "a", "b", "c"}, "/"), 0)
require.NoError(t, err)
_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot + "a"}, "/"))
_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot + "a"}, "/"), 0)
require.NoError(t, err)
_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot + "a", "b"}, "/"))
_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot + "a", "b"}, "/"), 0)
require.NoError(t, err)
})
@ -536,7 +536,7 @@ func TestShadowingExistingFolderByNestedContentRoot(t *testing.T) {
AllowUnsanitizedSvgUpload: true,
}
resp, err := store.List(ctx, globalUser, "content/nested")
resp, err := store.List(ctx, globalUser, "content/nested", 0)
require.NoError(t, err)
require.NotNil(t, resp)
@ -544,7 +544,7 @@ func TestShadowingExistingFolderByNestedContentRoot(t *testing.T) {
require.NoError(t, err)
require.Equal(t, 0, rowLen) // nested storage is empty
resp, err = store.List(ctx, globalUser, "content")
resp, err = store.List(ctx, globalUser, "content", 0)
require.NoError(t, err)
require.NotNil(t, resp)

18
pkg/services/store/tree.go
View File

@ -165,7 +165,7 @@ func (t *nestedTree) getStorages(orgId int64) []storageRuntime {
return storages
}
func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string, accessFilter filestorage.PathFilter) (*StorageListFrame, error) {
func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string, maxFiles int, accessFilter filestorage.PathFilter) (*StorageListFrame, error) {
if path == "" || path == "/" {
t.assureOrgIsInitialized(orgId)
@ -224,12 +224,16 @@ func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string, a
)
}
listResponse, err := store.List(ctx, path, nil, &filestorage.ListOptions{
Recursive: false,
WithFolders: true,
WithFiles: true,
Filter: pathFilter,
})
listResponse, err := store.List(ctx, path,
&filestorage.Paging{
Limit: maxFiles,
},
&filestorage.ListOptions{
Recursive: false,
WithFolders: true,
WithFiles: true,
Filter: pathFilter,
})
if err != nil {
return nil, err

2
pkg/services/store/types.go
View File

@ -40,7 +40,7 @@ type WriteValueResponse struct {
type storageTree interface {
GetFile(ctx context.Context, orgId int64, path string) (*filestorage.File, error)
ListFolder(ctx context.Context, orgId int64, path string, accessFilter filestorage.PathFilter) (*StorageListFrame, error)
ListFolder(ctx context.Context, orgId int64, path string, maxFiles int, accessFilter filestorage.PathFilter) (*StorageListFrame, error)
}
//-------------------------------------------

3
pkg/tsdb/grafanads/grafana.go
View File

@ -124,7 +124,8 @@ func (s *Service) doListQuery(ctx context.Context, query backend.DataQuery) back
}
path := store.RootPublicStatic + "/" + q.Path
listFrame, err := s.store.List(ctx, nil, path)
maxFiles := int(query.MaxDataPoints)
listFrame, err := s.store.List(ctx, nil, path, maxFiles)
response.Error = err
if listFrame != nil {
response.Frames = data.Frames{listFrame.Frame}

1
public/app/features/canvas/elements/icon.tsx
View File

@ -115,6 +115,7 @@ export const iconItem: CanvasElementItem<IconConfig, IconData> = {
editor: ResourceDimensionEditor,
settings: {
resourceType: 'icon',
maxFiles: 2000,
},
})
.addCustomEditor({

7
public/app/features/dimensions/editors/FolderPickerTab.tsx
View File

@ -35,10 +35,11 @@ interface Props {
folderName: ResourceFolderName;
newValue: string;
setNewValue: Dispatch<SetStateAction<string>>;
maxFiles?: number;
}
export const FolderPickerTab = (props: Props) => {
const { value, mediaType, folderName, newValue, setNewValue } = props;
const { value, mediaType, folderName, newValue, setNewValue, maxFiles } = props;
const styles = useStyles2(getStyles);
const folders = getFolders(mediaType).map((v) => ({
@ -75,7 +76,7 @@ export const FolderPickerTab = (props: Props) => {
getDatasourceSrv()
.get('-- Grafana --')
.then((ds) => {
(ds as GrafanaDatasource).listFiles(folder).subscribe({
(ds as GrafanaDatasource).listFiles(folder, maxFiles).subscribe({
next: (frame) => {
const cards: ResourceItem[] = [];
frame.forEach((item) => {
@ -95,7 +96,7 @@ export const FolderPickerTab = (props: Props) => {
});
});
}
}, [mediaType, currentFolder]);
}, [mediaType, currentFolder, maxFiles]);
return (
<>

2
public/app/features/dimensions/editors/ResourceDimensionEditor.tsx
View File

@ -65,6 +65,7 @@ export const ResourceDimensionEditor = (
const showSourceRadio = item.settings?.showSourceRadio ?? true;
const mediaType = item.settings?.resourceType ?? MediaType.Icon;
const folderName = item.settings?.folderName ?? ResourceFolderName.Icon;
const maxFiles = item.settings?.maxFiles; // undefined leads to backend default
let srcPath = '';
if (mediaType === MediaType.Icon) {
if (value?.fixed) {
@ -106,6 +107,7 @@ export const ResourceDimensionEditor = (
mediaType={mediaType}
folderName={folderName}
size={ResourcePickerSize.NORMAL}
maxFiles={maxFiles}
/>
)}
{mode === ResourceDimensionMode.Mapping && (

11
public/app/features/dimensions/editors/ResourcePicker.tsx
View File

@ -32,17 +32,24 @@ interface Props {
name?: string;
placeholder?: string;
color?: string;
maxFiles?: number;
}
export const ResourcePicker = (props: Props) => {
const { value, src, name, placeholder, onChange, onClear, mediaType, folderName, size, color } = props;
const { value, src, name, placeholder, onChange, onClear, mediaType, folderName, size, color, maxFiles } = props;
const styles = useStyles2(getStyles);
const theme = useTheme2();
const pickerTriggerRef = createRef<HTMLDivElement>();
const popoverElement = (
<ResourcePickerPopover onChange={onChange} value={value} mediaType={mediaType} folderName={folderName} />
<ResourcePickerPopover
onChange={onChange}
value={value}
mediaType={mediaType}
folderName={folderName}
maxFiles={maxFiles}
/>
);
let sanitizedSrc = src;

4
public/app/features/dimensions/editors/ResourcePickerPopover.tsx
View File

@ -20,13 +20,14 @@ interface Props {
onChange: (value?: string) => void;
mediaType: MediaType;
folderName: ResourceFolderName;
maxFiles?: number;
}
interface ErrorResponse {
message: string;
}
export const ResourcePickerPopover = (props: Props) => {
const { value, onChange, mediaType, folderName } = props;
const { value, onChange, mediaType, folderName, maxFiles } = props;
const styles = useStyles2(getStyles);
const onClose = () => {
@ -55,6 +56,7 @@ export const ResourcePickerPopover = (props: Props) => {
folderName={folderName}
newValue={newValue}
setNewValue={setNewValue}
maxFiles={maxFiles}
/>
);

1
public/app/features/dimensions/types.ts
View File

@ -59,6 +59,7 @@ export interface ResourceDimensionOptions {
placeholderValue?: string;
// If you want your icon to be driven by value of a field
showSourceRadio?: boolean;
maxFiles?: number;
}
export enum ResourceFolderName {

3
public/app/plugins/datasource/grafana/datasource.ts
View File

@ -170,7 +170,7 @@ export class GrafanaDatasource extends DataSourceWithBackend<GrafanaQuery> {
return of(); // nothing
}
listFiles(path: string): Observable<DataFrameView<FileElement>> {
listFiles(path: string, maxDataPoints?: number): Observable<DataFrameView<FileElement>> {
return this.query({
targets: [
{
@ -179,6 +179,7 @@ export class GrafanaDatasource extends DataSourceWithBackend<GrafanaQuery> {
path,
},
],
maxDataPoints,
} as any).pipe(
map((v) => {
const frame = v.data[0] ?? new MutableDataFrame();

3
public/app/plugins/panel/geomap/editor/StyleEditor.tsx
View File

@ -120,6 +120,7 @@ export const StyleEditor = (props: Props) => {
const propertyOptions = useObservable(settings?.layerInfo ?? of());
const featuresHavePoints = propertyOptions?.geometryType === GeometryTypeId.Point;
const hasTextLabel = styleUsesText(value);
const maxFiles = 2000;
// Simple fixed value display
if (settings?.simpleFixedValues) {
@ -141,6 +142,7 @@ export const StyleEditor = (props: Props) => {
placeholderText: hasTextLabel ? 'Select a symbol' : 'Select a symbol or add a text label',
placeholderValue: defaultStyleConfig.symbol.fixed,
showSourceRadio: false,
maxFiles,
},
} as StandardEditorsRegistryItem
}
@ -230,6 +232,7 @@ export const StyleEditor = (props: Props) => {
placeholderText: hasTextLabel ? 'Select a symbol' : 'Select a symbol or add a text label',
placeholderValue: defaultStyleConfig.symbol.fixed,
showSourceRadio: false,
maxFiles,
},
} as StandardEditorsRegistryItem
}