diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1a3db25c241..6e2be288b92 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -56,6 +56,7 @@ go.sum @grafana/backend-platform # Grafana edge /pkg/services/live/ @grafana/grafana-edge-squad /pkg/services/searchV2/ @grafana/grafana-edge-squad +/pkg/services/store/ @grafana/grafana-edge-squad /pkg/infra/filestore/ @grafana/grafana-edge-squad # Alerting diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 4cfe981b949..04137b73699 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -46,5 +46,7 @@ export interface FeatureToggles { dashboardComments?: boolean; annotationComments?: boolean; migrationLocking?: boolean; + storage?: boolean; + storageLocalUpload?: boolean; azureMonitorResourcePickerForMetrics?: boolean; } diff --git a/pkg/api/api.go b/pkg/api/api.go index 255c564c705..9f69e6f417f 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -211,6 +211,17 @@ func (hs *HTTPServer) registerRoutes() { orgRoute.Get("/quotas", authorize(reqSignedIn, ac.EvalPermission(ActionOrgsQuotasRead)), routing.Wrap(hs.GetCurrentOrgQuotas)) }) + if hs.Features.IsEnabled(featuremgmt.FlagStorage) { + apiRoute.Group("/storage", func(orgRoute routing.RouteRegister) { + orgRoute.Get("/list/", routing.Wrap(hs.StorageService.List)) + orgRoute.Get("/list/*", routing.Wrap(hs.StorageService.List)) + orgRoute.Get("/read/*", routing.Wrap(hs.StorageService.Read)) + + orgRoute.Delete("/delete/*", reqSignedIn, routing.Wrap(hs.StorageService.Delete)) + orgRoute.Post("/upload", reqSignedIn, routing.Wrap(hs.StorageService.Upload)) + }) + } + // current org apiRoute.Group("/org", func(orgRoute routing.RouteRegister) { userIDScope := ac.Scope("users", "id", ac.Parameter(":userId")) diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index e2823d6456b..91aee6e9c2a 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -62,6 +62,7 @@ import ( "github.com/grafana/grafana/pkg/services/serviceaccounts" "github.com/grafana/grafana/pkg/services/shorturls" "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/services/store" "github.com/grafana/grafana/pkg/services/teamguardian" "github.com/grafana/grafana/pkg/services/thumbs" "github.com/grafana/grafana/pkg/services/updatechecker" @@ -110,6 +111,7 @@ type HTTPServer struct { Live *live.GrafanaLive LivePushGateway *pushhttp.Gateway ThumbService thumbs.Service + StorageService store.HTTPStorageService ContextHandler *contexthandler.ContextHandler SQLStore sqlstore.Store AlertEngine *alerting.AlertEngine @@ -169,7 +171,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi pluginsUpdateChecker *updatechecker.PluginsService, searchUsersService searchusers.Service, dataSourcesService datasources.DataSourceService, secretsService secrets.Service, queryDataService *query.Service, ldapGroups ldap.Groups, teamGuardian teamguardian.TeamGuardian, serviceaccountsService serviceaccounts.Service, - authInfoService login.AuthInfoService, permissionsServices accesscontrol.PermissionsServices, + authInfoService login.AuthInfoService, permissionsServices accesscontrol.PermissionsServices, storageService store.HTTPStorageService, notificationService *notifications.NotificationService, dashboardService dashboards.DashboardService, dashboardProvisioningService dashboards.DashboardProvisioningService, folderService dashboards.FolderService, datasourcePermissionsService permissions.DatasourcePermissionsService, alertNotificationService *alerting.AlertNotificationService, @@ -204,6 +206,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi QueryHistoryService: queryHistoryService, Features: features, ThumbService: thumbService, + StorageService: storageService, RemoteCacheService: remoteCache, ProvisioningService: provisioningService, Login: loginService, diff --git a/pkg/infra/filestorage/cdk_blob_filestorage.go b/pkg/infra/filestorage/cdk_blob_filestorage.go index a49ab1ebeec..27a5f4aac03 100644 --- a/pkg/infra/filestorage/cdk_blob_filestorage.go +++ b/pkg/infra/filestorage/cdk_blob_filestorage.go @@ -391,7 +391,7 @@ func (c cdkBlobStorage) list(ctx context.Context, folderPath string, paging *Pag hasMore := false if len(files) > pageSize { hasMore = true - files = files[:len(files)-pageSize] + files = files[:pageSize] } lastPath := "" diff --git a/pkg/infra/filestorage/fs_integration_test.go b/pkg/infra/filestorage/fs_integration_test.go index a9b8b21581c..5b9268497ff 100644 --- a/pkg/infra/filestorage/fs_integration_test.go +++ b/pkg/infra/filestorage/fs_integration_test.go @@ -427,17 +427,19 @@ func TestFsStorage(t *testing.T) { }, }, queryListFiles{ - input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: ""}}, - list: checks(listSize(1), listHasMore(true), listLastPath("/folder1/a")), + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 2, After: ""}}, + list: checks(listSize(2), listHasMore(true), listLastPath("/folder1/b")), files: [][]interface{}{ checks(fPath("/folder1/a")), + checks(fPath("/folder1/b")), }, }, queryListFiles{ - input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: ""}}, - list: checks(listSize(1), listHasMore(true), listLastPath("/folder1")), + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 2, After: ""}}, + list: checks(listSize(2), listHasMore(true), listLastPath("/folder1/a")), files: [][]interface{}{ checks(fPath("/folder1"), fMimeType(DirectoryMimeType)), + checks(fPath("/folder1/a")), }, }, queryListFiles{ diff --git a/pkg/plugins/manager/manager_integration_test.go b/pkg/plugins/manager/manager_integration_test.go index 1bb2651f8d2..fe61b232729 100644 --- a/pkg/plugins/manager/manager_integration_test.go +++ b/pkg/plugins/manager/manager_integration_test.go @@ -87,7 +87,7 @@ func TestPluginManager_int_init(t *testing.T) { my := mysql.ProvideService(cfg, hcp) ms := mssql.ProvideService(cfg) sv2 := searchV2.ProvideService(sqlstore.InitTestDB(t)) - graf := grafanads.ProvideService(cfg, sv2) + graf := grafanads.ProvideService(cfg, sv2, nil) coreRegistry := coreplugin.ProvideCoreRegistry(am, cw, cm, es, grap, idb, lk, otsdb, pr, tmpo, td, pg, my, ms, graf) diff --git a/pkg/server/backgroundsvcs/background_services.go b/pkg/server/backgroundsvcs/background_services.go index 4ec2356c23d..8521761435e 100644 --- a/pkg/server/backgroundsvcs/background_services.go +++ b/pkg/server/backgroundsvcs/background_services.go @@ -22,6 +22,7 @@ import ( "github.com/grafana/grafana/pkg/services/rendering" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/services/serviceaccounts" + "github.com/grafana/grafana/pkg/services/store" "github.com/grafana/grafana/pkg/services/thumbs" "github.com/grafana/grafana/pkg/services/updatechecker" ) @@ -33,7 +34,7 @@ func ProvideBackgroundServiceRegistry( provisioning *provisioning.ProvisioningServiceImpl, alerting *alerting.AlertEngine, usageStats *uss.UsageStats, grafanaUpdateChecker *updatechecker.GrafanaService, pluginsUpdateChecker *updatechecker.PluginsService, metrics *metrics.InternalMetricsService, secretsService *secretsManager.SecretsService, - remoteCache *remotecache.RemoteCache, thumbnailsService thumbs.Service, + remoteCache *remotecache.RemoteCache, thumbnailsService thumbs.Service, StorageService store.StorageService, // Need to make sure these are initialized, is there a better place to put them? _ *dashboardsnapshots.Service, _ *alerting.AlertNotificationService, _ serviceaccounts.Service, _ *guardian.Provider, @@ -58,6 +59,7 @@ func ProvideBackgroundServiceRegistry( tracing, remoteCache, secretsService, + StorageService, thumbnailsService) } diff --git a/pkg/server/wire.go b/pkg/server/wire.go index e610f22f19a..48acb1ca973 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -75,6 +75,7 @@ import ( "github.com/grafana/grafana/pkg/services/shorturls" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/sqlstore/mockstore" + "github.com/grafana/grafana/pkg/services/store" "github.com/grafana/grafana/pkg/services/teamguardian" teamguardianDatabase "github.com/grafana/grafana/pkg/services/teamguardian/database" teamguardianManager "github.com/grafana/grafana/pkg/services/teamguardian/manager" @@ -158,6 +159,8 @@ var wireBasicSet = wire.NewSet( datasourceproxy.ProvideService, search.ProvideService, searchV2.ProvideService, + store.ProvideService, + store.ProvideHTTPService, live.ProvideService, pushhttp.ProvideService, plugincontext.ProvideService, diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 3dc7a0b675a..d9bebce3bfd 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -162,6 +162,17 @@ var ( Description: "Lock database during migrations", State: FeatureStateBeta, }, + { + Name: "storage", + Description: "Configurable storage for dashboards, datasources, and resources", + State: FeatureStateAlpha, + }, + { + Name: "storageLocalUpload", + Description: "allow uploads to local storage", + State: FeatureStateAlpha, + RequiresDevMode: true, + }, { Name: "azureMonitorResourcePickerForMetrics", Description: "New UI for Azure Monitor Metrics Query", diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index ba09cd822ca..4cbd538088c 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -123,6 +123,14 @@ const ( // Lock database during migrations FlagMigrationLocking = "migrationLocking" + // FlagStorage + // Configurable storage for dashboards, datasources, and resources + FlagStorage = "storage" + + // FlagStorageLocalUpload + // allow uploads to local storage + FlagStorageLocalUpload = "storageLocalUpload" + // FlagAzureMonitorResourcePickerForMetrics // New UI for Azure Monitor Metrics Query FlagAzureMonitorResourcePickerForMetrics = "azureMonitorResourcePickerForMetrics" diff --git a/pkg/services/store/config.go b/pkg/services/store/config.go new file mode 100644 index 00000000000..6ae3b132042 --- /dev/null +++ b/pkg/services/store/config.go @@ -0,0 +1,53 @@ +package store + +type RootStorageConfig struct { + Type string `json:"type"` + Prefix string `json:"prefix"` + Name string `json:"name"` + + // Depending on type, these will be configured + Disk *StorageLocalDiskConfig `json:"disk,omitempty"` + Git *StorageGitConfig `json:"git,omitempty"` + SQL *StorageSQLConfig `json:"sql,omitempty"` + S3 *StorageS3Config `json:"s3,omitempty"` + GCS *StorageGCSConfig `json:"gcs,omitempty"` +} + +type StorageLocalDiskConfig struct { + Path string `json:"path"` + Roots []string `json:"roots,omitempty"` // null is everything +} + +type StorageGitConfig struct { + Remote string `json:"remote"` + Branch string `json:"branch"` + Root string `json:"root"` // subfolder within the remote + + // Pull interval? + // Requires pull request? + RequirePullRequest bool `json:"requirePullRequest"` + + // SECURE JSON :grimicing: + AccessToken string `json:"accessToken,omitempty"` // Simplest auth method for github +} + +type StorageSQLConfig struct { + // no custom settings +} + +type StorageS3Config struct { + Bucket string `json:"bucket"` + Folder string `json:"folder"` + + // SECURE!!! + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` + Region string `json:"region"` +} + +type StorageGCSConfig struct { + Bucket string `json:"bucket"` + Folder string `json:"folder"` + + CredentialsFile string `json:"credentialsFile"` +} diff --git a/pkg/services/store/http.go b/pkg/services/store/http.go new file mode 100644 index 00000000000..5d23128a8cf --- /dev/null +++ b/pkg/services/store/http.go @@ -0,0 +1,71 @@ +package store + +import ( + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/web" +) + +// HTTPStorageService passes raw HTTP requests to a well typed storage service +type HTTPStorageService interface { + List(c *models.ReqContext) response.Response + Read(c *models.ReqContext) response.Response + Delete(c *models.ReqContext) response.Response + Upload(c *models.ReqContext) response.Response +} + +type httpStorage struct { + store StorageService +} + +func ProvideHTTPService(store StorageService) HTTPStorageService { + return &httpStorage{ + store: store, + } +} + +func (s *httpStorage) Upload(c *models.ReqContext) response.Response { + action := "Upload" + scope, path := getPathAndScope(c) + + return response.JSON(200, map[string]string{ + "action": action, + "scope": scope, + "path": path, + }) +} + +func (s *httpStorage) Read(c *models.ReqContext) response.Response { + action := "Read" + scope, path := getPathAndScope(c) + + return response.JSON(200, map[string]string{ + "action": action, + "scope": scope, + "path": path, + }) +} + +func (s *httpStorage) Delete(c *models.ReqContext) response.Response { + action := "Delete" + scope, path := getPathAndScope(c) + + return response.JSON(200, map[string]string{ + "action": action, + "scope": scope, + "path": path, + }) +} + +func (s *httpStorage) List(c *models.ReqContext) response.Response { + params := web.Params(c.Req) + path := params["*"] + frame, err := s.store.List(c.Req.Context(), c.SignedInUser, path) + if err != nil { + return response.Error(400, "error reading path", err) + } + if frame == nil { + return response.Error(404, "not found", nil) + } + return response.JSONStreaming(200, frame) +} diff --git a/pkg/services/store/service.go b/pkg/services/store/service.go new file mode 100644 index 00000000000..42f9fcf449b --- /dev/null +++ b/pkg/services/store/service.go @@ -0,0 +1,88 @@ +package store + +import ( + "context" + "os" + "path/filepath" + + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/filestorage" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/registry" + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/setting" +) + +var grafanaStorageLogger = log.New("grafanaStorageLogger") + +const RootPublicStatic = "public-static" + +type StorageService interface { + registry.BackgroundService + + // List folder contents + List(ctx context.Context, user *models.SignedInUser, path string) (*data.Frame, error) + + // Read raw file contents out of the store + Read(ctx context.Context, user *models.SignedInUser, path string) (*filestorage.File, error) +} + +type standardStorageService struct { + sql *sqlstore.SQLStore + tree *nestedTree +} + +func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, cfg *setting.Cfg) StorageService { + roots := []storageRuntime{ + newDiskStorage(RootPublicStatic, "Public static files", &StorageLocalDiskConfig{ + Path: cfg.StaticRootPath, + Roots: []string{ + "/testdata/", + // "/img/icons/", + // "/img/bg/", + "/img/", + "/gazetteer/", + "/maps/", + }, + }).setReadOnly(true).setBuiltin(true), + } + + storage := filepath.Join(cfg.DataPath, "storage") + _ = os.MkdirAll(storage, 0700) + + if features.IsEnabled(featuremgmt.FlagStorageLocalUpload) { + roots = append(roots, newDiskStorage("upload", "Local file upload", &StorageLocalDiskConfig{ + Path: filepath.Join(storage, "upload"), + })) + } + s := newStandardStorageService(roots) + s.sql = sql + return s +} + +func newStandardStorageService(roots []storageRuntime) *standardStorageService { + res := &nestedTree{ + roots: roots, + } + res.init() + return &standardStorageService{ + tree: res, + } +} + +func (s *standardStorageService) Run(ctx context.Context) error { + grafanaStorageLogger.Info("storage starting") + return nil +} + +func (s *standardStorageService) List(ctx context.Context, user *models.SignedInUser, path string) (*data.Frame, error) { + // apply access control here + return s.tree.ListFolder(ctx, path) +} + +func (s *standardStorageService) Read(ctx context.Context, user *models.SignedInUser, path string) (*filestorage.File, error) { + // TODO: permission check! + return s.tree.GetFile(ctx, path) +} diff --git a/pkg/services/store/service_test.go b/pkg/services/store/service_test.go new file mode 100644 index 00000000000..f2b38d0cede --- /dev/null +++ b/pkg/services/store/service_test.go @@ -0,0 +1,47 @@ +package store + +import ( + "bytes" + "context" + "path" + "path/filepath" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/experimental" + "github.com/grafana/grafana/pkg/tsdb/testdatasource" + "github.com/stretchr/testify/require" +) + +func TestListFiles(t *testing.T) { + publicRoot, err := filepath.Abs("../../../public") + require.NoError(t, err) + roots := []storageRuntime{ + newDiskStorage("public", "Public static files", &StorageLocalDiskConfig{ + Path: publicRoot, + Roots: []string{ + "/testdata/", + "/img/icons/", + "/img/bg/", + "/gazetteer/", + "/maps/", + "/upload/", + }, + }).setReadOnly(true).setBuiltin(true), + } + + store := newStandardStorageService(roots) + frame, err := store.List(context.Background(), nil, "public/testdata") + require.NoError(t, err) + + err = experimental.CheckGoldenFrame(path.Join("testdata", "public_testdata.golden.txt"), frame, true) + require.NoError(t, err) + + file, err := store.Read(context.Background(), nil, "public/testdata/js_libraries.csv") + require.NoError(t, err) + require.NotNil(t, file) + + frame, err = testdatasource.LoadCsvContent(bytes.NewReader(file.Contents), file.Name) + require.NoError(t, err) + err = experimental.CheckGoldenFrame(path.Join("testdata", "public_testdata_js_libraries.golden.txt"), frame, true) + require.NoError(t, err) +} diff --git a/pkg/services/store/storage_disk.go b/pkg/services/store/storage_disk.go new file mode 100644 index 00000000000..e7dde055629 --- /dev/null +++ b/pkg/services/store/storage_disk.go @@ -0,0 +1,91 @@ +package store + +import ( + "context" + "fmt" + "strings" + + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/filestorage" + "gocloud.dev/blob" +) + +const rootStorageTypeDisk = "disk" + +type rootStorageDisk struct { + baseStorageRuntime + + settings *StorageLocalDiskConfig +} + +func newDiskStorage(prefix string, name string, cfg *StorageLocalDiskConfig) *rootStorageDisk { + if cfg == nil { + cfg = &StorageLocalDiskConfig{} + } + + meta := RootStorageMeta{ + Config: RootStorageConfig{ + Type: rootStorageTypeDisk, + Prefix: prefix, + Name: name, + Disk: cfg, + }, + } + if prefix == "" { + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: "Missing prefix", + }) + } + if cfg.Path == "" { + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: "Missing path configuration", + }) + } + s := &rootStorageDisk{} + + if meta.Notice == nil { + path := fmt.Sprintf("file://%s", cfg.Path) + bucket, err := blob.OpenBucket(context.Background(), path) + if err != nil { + grafanaStorageLogger.Warn("error loading storage", "prefix", prefix, "err", err) + meta.Notice = append(meta.Notice, data.Notice{ + Severity: data.NoticeSeverityError, + Text: "Failed to initialize storage", + }) + } else { + s.store = filestorage.NewCdkBlobStorage(grafanaStorageLogger, + bucket, "", + filestorage.NewPathFilters(cfg.Roots, nil, nil, nil)) + + meta.Ready = true // exists! + } + } + + s.meta = meta + s.settings = cfg + return s +} + +func (s *rootStorageDisk) Sync() error { + return nil // already in sync +} + +// with local disk user metadata and messages are lost +func (s *rootStorageDisk) Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error) { + byteAray := []byte(cmd.Body) + + path := cmd.Path + if !strings.HasPrefix(path, filestorage.Delimiter) { + path = filestorage.Delimiter + path + } + err := s.store.Upsert(ctx, &filestorage.UpsertFileCommand{ + Path: path, + Contents: &byteAray, + }) + if err != nil { + return nil, err + } + return &WriteValueResponse{Code: 200}, nil +} diff --git a/pkg/services/store/testdata/public_testdata.golden.txt b/pkg/services/store/testdata/public_testdata.golden.txt new file mode 100644 index 00000000000..7bb0c6a52e1 --- /dev/null +++ b/pkg/services/store/testdata/public_testdata.golden.txt @@ -0,0 +1,27 @@ +🌟 This was machine generated. Do not edit. 🌟 + +Frame[0] { + "type": "directory-listing", + "custom": { + "HasMore": false + } +} +Name: +Dimensions: 3 Fields by 7 Rows ++--------------------------+-------------------------+---------------+ +| Name: name | Name: mediaType | Name: size | +| Labels: | Labels: | Labels: | +| Type: []string | Type: []string | Type: []int64 | ++--------------------------+-------------------------+---------------+ +| browser_marketshare.csv | text/csv; charset=utf-8 | 355 | +| flight_info_by_state.csv | text/csv; charset=utf-8 | 681 | +| gdp_per_capita.csv | text/csv; charset=utf-8 | 4116 | +| js_libraries.csv | text/csv; charset=utf-8 | 179 | +| ohlc_dogecoin.csv | text/csv; charset=utf-8 | 191804 | +| population_by_state.csv | text/csv; charset=utf-8 | 138 | +| weight_height.csv | text/csv; charset=utf-8 | 418121 | ++--------------------------+-------------------------+---------------+ + + +====== TEST DATA RESPONSE (arrow base64) ====== +FRAME=QVJST1cxAAD/////WAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAKgAAAADAAAATAAAACgAAAAEAAAAMP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABQ/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAHD+//8IAAAAQAAAADcAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsImN1c3RvbSI6eyJIYXNNb3JlIjpmYWxzZX19AAQAAABtZXRhAAAAAAMAAAAYAQAApAAAAAQAAAAG////FAAAAHAAAAB4AAAAAAAAAnwAAAACAAAALAAAAAQAAAD4/v//CAAAABAAAAAEAAAAc2l6ZQAAAAAEAAAAbmFtZQAAAAAc////CAAAABwAAAAQAAAAeyJ1bml0IjoiYnl0ZXMifQAAAAAGAAAAY29uZmlnAAAAAAAACAAMAAgABwAIAAAAAAAAAUAAAAAEAAAAc2l6ZQAAAACi////FAAAAEAAAABAAAAAAAAABTwAAAABAAAABAAAAJD///8IAAAAFAAAAAkAAABtZWRpYVR5cGUAAAAEAAAAbmFtZQAAAAAAAAAAjP///wkAAABtZWRpYVR5cGUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABuYW1lAAAAAAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP////8IAQAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAsAEAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAmAAAAAcAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAigAAAAAAAACwAAAAAAAAAAAAAAAAAAAAsAAAAAAAAAAgAAAAAAAAANAAAAAAAAAAoQAAAAAAAAB4AQAAAAAAAAAAAAAAAAAAeAEAAAAAAAA4AAAAAAAAAAAAAAADAAAABwAAAAAAAAAAAAAAAAAAAAcAAAAAAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAABcAAAAvAAAAQQAAAFEAAABiAAAAeQAAAIoAAABicm93c2VyX21hcmtldHNoYXJlLmNzdmZsaWdodF9pbmZvX2J5X3N0YXRlLmNzdmdkcF9wZXJfY2FwaXRhLmNzdmpzX2xpYnJhcmllcy5jc3ZvaGxjX2RvZ2Vjb2luLmNzdnBvcHVsYXRpb25fYnlfc3RhdGUuY3N2d2VpZ2h0X2hlaWdodC5jc3YAAAAAAAAAAAAAFwAAAC4AAABFAAAAXAAAAHMAAACKAAAAoQAAAHRleHQvY3N2OyBjaGFyc2V0PXV0Zi04dGV4dC9jc3Y7IGNoYXJzZXQ9dXRmLTh0ZXh0L2NzdjsgY2hhcnNldD11dGYtOHRleHQvY3N2OyBjaGFyc2V0PXV0Zi04dGV4dC9jc3Y7IGNoYXJzZXQ9dXRmLTh0ZXh0L2NzdjsgY2hhcnNldD11dGYtOHRleHQvY3N2OyBjaGFyc2V0PXV0Zi04AAAAAAAAAGMBAAAAAAAAqQIAAAAAAAAUEAAAAAAAALMAAAAAAAAAPO0CAAAAAACKAAAAAAAAAElhBgAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAQAAQAAAGgCAAAAAAAAEAEAAAAAAACwAQAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAqAAAAAMAAABMAAAAKAAAAAQAAAAw/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAFD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAcP7//wgAAABAAAAANwAAAHsidHlwZSI6ImRpcmVjdG9yeS1saXN0aW5nIiwiY3VzdG9tIjp7Ikhhc01vcmUiOmZhbHNlfX0ABAAAAG1ldGEAAAAAAwAAABgBAACkAAAABAAAAAb///8UAAAAcAAAAHgAAAAAAAACfAAAAAIAAAAsAAAABAAAAPj+//8IAAAAEAAAAAQAAABzaXplAAAAAAQAAABuYW1lAAAAABz///8IAAAAHAAAABAAAAB7InVuaXQiOiJieXRlcyJ9AAAAAAYAAABjb25maWcAAAAAAAAIAAwACAAHAAgAAAAAAAABQAAAAAQAAABzaXplAAAAAKL///8UAAAAQAAAAEAAAAAAAAAFPAAAAAEAAAAEAAAAkP///wgAAAAUAAAACQAAAG1lZGlhVHlwZQAAAAQAAABuYW1lAAAAAAAAAACM////CQAAAG1lZGlhVHlwZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAG5hbWUAAAAABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAABAAAAG5hbWUAAAAAiAIAAEFSUk9XMQ== diff --git a/pkg/tsdb/grafanads/testdata/jslib.golden.txt b/pkg/services/store/testdata/public_testdata_js_libraries.golden.txt similarity index 92% rename from pkg/tsdb/grafanads/testdata/jslib.golden.txt rename to pkg/services/store/testdata/public_testdata_js_libraries.golden.txt index 08f17e7b79e..0d9d7ed4a38 100644 --- a/pkg/tsdb/grafanads/testdata/jslib.golden.txt +++ b/pkg/services/store/testdata/public_testdata_js_libraries.golden.txt @@ -18,4 +18,4 @@ Dimensions: 4 Fields by 6 Rows ====== TEST DATA RESPONSE (arrow base64) ====== -FRAME=QVJST1cxAAD/////WAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAGAAAAACAAAAKAAAAAQAAAAs/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAEz+//8IAAAAHAAAABAAAABqc19saWJyYXJpZXMuY3N2AAAAAAQAAABuYW1lAAAAAAQAAABgAQAA1AAAAHAAAAAEAAAAwv7//xQAAABAAAAAQAAAAAAAAgFEAAAAAQAAAAQAAACw/v//CAAAABQAAAAIAAAAV2F0Y2hlcnMAAAAABAAAAG5hbWUAAAAAAAAAADT///8AAAABQAAAAAgAAABXYXRjaGVycwAAAAAq////FAAAADwAAAA8AAAAAAACAUAAAAABAAAABAAAABj///8IAAAAEAAAAAUAAABGb3JrcwAAAAQAAABuYW1lAAAAAAAAAACY////AAAAAUAAAAAFAAAARm9ya3MAAACK////FAAAAEQAAABMAAAAAAACAVAAAAABAAAABAAAAHj///8IAAAAGAAAAAwAAABHaXRodWIgU3RhcnMAAAAABAAAAG5hbWUAAAAAAAAAAAgADAAIAAcACAAAAAAAAAFAAAAADAAAAEdpdGh1YiBTdGFycwAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAABQFEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAcAAABMaWJyYXJ5AAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAcAAABMaWJyYXJ5AP////8oAQAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAA2AAAAAAAAAAUAAAAAAAAAwMACgAYAAwACAAEAAoAAAAUAAAAqAAAAAYAAAAAAAAAAAAAAAkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAKAAAAAAAAABIAAAAAAAAAAAAAAAAAAAASAAAAAAAAAAwAAAAAAAAAHgAAAAAAAAAAAAAAAAAAAB4AAAAAAAAADAAAAAAAAAAqAAAAAAAAAAAAAAAAAAAAKgAAAAAAAAAMAAAAAAAAAAAAAAABAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAsAAAASAAAAGAAAAB4AAAAlAAAAAAAAAFJlYWN0LmpzVnVlQW5ndWxhckpRdWVyeU1ldGVvckF1cmVsaWEAAAAolAIAAAAAAMDOAgAAAAAAuB4BAAAAAAB01gAAAAAAAKClAAAAAAAAUC0AAAAAAADQhAAAAAAAAKxxAAAAAAAAZEsAAAAAAAAgTgAAAAAAAFAUAAAAAAAArAIAAAAAAAAsGgAAAAAAAJwYAAAAAAAAgAwAAAAAAADkDAAAAAAAAKQGAAAAAAAAugEAAAAAAAAQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAAAwABAAAAaAIAAAAAAAAwAQAAAAAAANgAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABgAAAAAgAAACgAAAAEAAAALP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABM/v//CAAAABwAAAAQAAAAanNfbGlicmFyaWVzLmNzdgAAAAAEAAAAbmFtZQAAAAAEAAAAYAEAANQAAABwAAAABAAAAML+//8UAAAAQAAAAEAAAAAAAAIBRAAAAAEAAAAEAAAAsP7//wgAAAAUAAAACAAAAFdhdGNoZXJzAAAAAAQAAABuYW1lAAAAAAAAAAA0////AAAAAUAAAAAIAAAAV2F0Y2hlcnMAAAAAKv///xQAAAA8AAAAPAAAAAAAAgFAAAAAAQAAAAQAAAAY////CAAAABAAAAAFAAAARm9ya3MAAAAEAAAAbmFtZQAAAAAAAAAAmP///wAAAAFAAAAABQAAAEZvcmtzAAAAiv///xQAAABEAAAATAAAAAAAAgFQAAAAAQAAAAQAAAB4////CAAAABgAAAAMAAAAR2l0aHViIFN0YXJzAAAAAAQAAABuYW1lAAAAAAAAAAAIAAwACAAHAAgAAAAAAAABQAAAAAwAAABHaXRodWIgU3RhcnMAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAARAAAAEgAAAAAAAUBRAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAHAAAATGlicmFyeQAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAHAAAATGlicmFyeQCIAgAAQVJST1cx +FRAME=QVJST1cxAAD/////WAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAGAAAAACAAAAKAAAAAQAAAAs/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAEz+//8IAAAAHAAAABAAAABqc19saWJyYXJpZXMuY3N2AAAAAAQAAABuYW1lAAAAAAQAAABgAQAA1AAAAHAAAAAEAAAAwv7//xQAAABAAAAAQAAAAAAAAgFEAAAAAQAAAAQAAACw/v//CAAAABQAAAAIAAAAV2F0Y2hlcnMAAAAABAAAAG5hbWUAAAAAAAAAADT///8AAAABQAAAAAgAAABXYXRjaGVycwAAAAAq////FAAAADwAAAA8AAAAAAACAUAAAAABAAAABAAAABj///8IAAAAEAAAAAUAAABGb3JrcwAAAAQAAABuYW1lAAAAAAAAAACY////AAAAAUAAAAAFAAAARm9ya3MAAACK////FAAAAEQAAABMAAAAAAACAVAAAAABAAAABAAAAHj///8IAAAAGAAAAAwAAABHaXRodWIgU3RhcnMAAAAABAAAAG5hbWUAAAAAAAAAAAgADAAIAAcACAAAAAAAAAFAAAAADAAAAEdpdGh1YiBTdGFycwAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAABQFEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAcAAABMaWJyYXJ5AAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAcAAABMaWJyYXJ5AP////8oAQAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAA2AAAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAqAAAAAYAAAAAAAAAAAAAAAkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAcAAAAAAAAACAAAAAAAAAAJQAAAAAAAABIAAAAAAAAAAAAAAAAAAAASAAAAAAAAAAwAAAAAAAAAHgAAAAAAAAAAAAAAAAAAAB4AAAAAAAAADAAAAAAAAAAqAAAAAAAAAAAAAAAAAAAAKgAAAAAAAAAMAAAAAAAAAAAAAAABAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAsAAAASAAAAGAAAAB4AAAAlAAAAAAAAAFJlYWN0LmpzVnVlQW5ndWxhckpRdWVyeU1ldGVvckF1cmVsaWEAAAAolAIAAAAAAMDOAgAAAAAAuB4BAAAAAAB01gAAAAAAAKClAAAAAAAAUC0AAAAAAADQhAAAAAAAAKxxAAAAAAAAZEsAAAAAAAAgTgAAAAAAAFAUAAAAAAAArAIAAAAAAAAsGgAAAAAAAJwYAAAAAAAAgAwAAAAAAADkDAAAAAAAAKQGAAAAAAAAugEAAAAAAAAQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAABAABAAAAaAIAAAAAAAAwAQAAAAAAANgAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABgAAAAAgAAACgAAAAEAAAALP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABM/v//CAAAABwAAAAQAAAAanNfbGlicmFyaWVzLmNzdgAAAAAEAAAAbmFtZQAAAAAEAAAAYAEAANQAAABwAAAABAAAAML+//8UAAAAQAAAAEAAAAAAAAIBRAAAAAEAAAAEAAAAsP7//wgAAAAUAAAACAAAAFdhdGNoZXJzAAAAAAQAAABuYW1lAAAAAAAAAAA0////AAAAAUAAAAAIAAAAV2F0Y2hlcnMAAAAAKv///xQAAAA8AAAAPAAAAAAAAgFAAAAAAQAAAAQAAAAY////CAAAABAAAAAFAAAARm9ya3MAAAAEAAAAbmFtZQAAAAAAAAAAmP///wAAAAFAAAAABQAAAEZvcmtzAAAAiv///xQAAABEAAAATAAAAAAAAgFQAAAAAQAAAAQAAAB4////CAAAABgAAAAMAAAAR2l0aHViIFN0YXJzAAAAAAQAAABuYW1lAAAAAAAAAAAIAAwACAAHAAgAAAAAAAABQAAAAAwAAABHaXRodWIgU3RhcnMAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAARAAAAEgAAAAAAAUBRAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAHAAAATGlicmFyeQAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAHAAAATGlicmFyeQCIAgAAQVJST1cx diff --git a/pkg/services/store/tree.go b/pkg/services/store/tree.go new file mode 100644 index 00000000000..7e3dd20ff7d --- /dev/null +++ b/pkg/services/store/tree.go @@ -0,0 +1,106 @@ +package store + +import ( + "context" + + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/filestorage" +) + +type nestedTree struct { + roots []storageRuntime + lookup map[string]filestorage.FileStorage +} + +var ( + _ storageTree = (*nestedTree)(nil) +) + +func (t *nestedTree) init() { + t.lookup = make(map[string]filestorage.FileStorage, len(t.roots)) + for _, root := range t.roots { + t.lookup[root.Meta().Config.Prefix] = root.Store() + } +} + +func (t *nestedTree) getRoot(path string) (filestorage.FileStorage, string) { + if path == "" { + return nil, "" + } + + rootKey, path := splitFirstSegment(path) + root, ok := t.lookup[rootKey] + if !ok || root == nil { + return nil, path // not found or not ready + } + return root, filestorage.Delimiter + path +} + +func (t *nestedTree) GetFile(ctx context.Context, path string) (*filestorage.File, error) { + if path == "" { + return nil, nil // not found + } + + root, path := t.getRoot(path) + if root == nil { + return nil, nil // not found (or not ready) + } + return root.Get(ctx, path) +} + +func (t *nestedTree) ListFolder(ctx context.Context, path string) (*data.Frame, error) { + if path == "" || path == "/" { + count := len(t.roots) + title := data.NewFieldFromFieldType(data.FieldTypeString, count) + names := data.NewFieldFromFieldType(data.FieldTypeString, count) + mtype := data.NewFieldFromFieldType(data.FieldTypeString, count) + title.Name = "title" + names.Name = "name" + mtype.Name = "mediaType" + for i, f := range t.roots { + names.Set(i, f.Meta().Config.Prefix) + title.Set(i, f.Meta().Config.Name) + mtype.Set(i, "directory") + } + frame := data.NewFrame("", names, title, mtype) + frame.SetMeta(&data.FrameMeta{ + Type: data.FrameTypeDirectoryListing, + }) + return frame, nil + } + + root, path := t.getRoot(path) + if root == nil { + return nil, nil // not found (or not ready) + } + + listResponse, err := root.List(ctx, path, nil, &filestorage.ListOptions{Recursive: false, WithFolders: true, WithFiles: true}) + + if err != nil { + return nil, err + } + + count := len(listResponse.Files) + names := data.NewFieldFromFieldType(data.FieldTypeString, count) + mtype := data.NewFieldFromFieldType(data.FieldTypeString, count) + fsize := data.NewFieldFromFieldType(data.FieldTypeInt64, count) + names.Name = "name" + mtype.Name = "mediaType" + fsize.Name = "size" + fsize.Config = &data.FieldConfig{ + Unit: "bytes", + } + for i, f := range listResponse.Files { + names.Set(i, f.Name) + mtype.Set(i, f.MimeType) + fsize.Set(i, f.Size) + } + frame := data.NewFrame("", names, mtype, fsize) + frame.SetMeta(&data.FrameMeta{ + Type: data.FrameTypeDirectoryListing, + Custom: map[string]interface{}{ + "HasMore": listResponse.HasMore, + }, + }) + return frame, nil +} diff --git a/pkg/services/store/types.go b/pkg/services/store/types.go new file mode 100644 index 00000000000..f280dcfc641 --- /dev/null +++ b/pkg/services/store/types.go @@ -0,0 +1,92 @@ +package store + +import ( + "context" + "encoding/json" + + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/filestorage" + "github.com/grafana/grafana/pkg/models" +) + +type WriteValueRequest struct { + Path string + User *models.SignedInUser + Body json.RawMessage `json:"body,omitempty"` + Message string `json:"message,omitempty"` + Title string `json:"title,omitempty"` // For PRs + Action string `json:"action,omitempty"` // pr | save +} + +type WriteValueResponse struct { + Code int `json:"code,omitempty"` + Message string `json:"message,omitempty"` + URL string `json:"url,omitempty"` + Hash string `json:"hash,omitempty"` + Branch string `json:"branch,omitempty"` + Pending bool `json:"pending,omitempty"` + Size int64 `json:"size,omitempty"` +} + +type storageTree interface { + GetFile(ctx context.Context, path string) (*filestorage.File, error) + ListFolder(ctx context.Context, path string) (*data.Frame, error) +} + +//------------------------------------------- +// INTERNAL +//------------------------------------------- + +type storageRuntime interface { + Meta() RootStorageMeta + + Store() filestorage.FileStorage + + Sync() error + + // Different storage knows how to handle comments and tracking + Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error) +} + +type baseStorageRuntime struct { + meta RootStorageMeta + store filestorage.FileStorage +} + +func (t *baseStorageRuntime) Meta() RootStorageMeta { + return t.meta +} + +func (t *baseStorageRuntime) Store() filestorage.FileStorage { + return t.store +} + +func (t *baseStorageRuntime) Sync() error { + return nil +} + +func (t *baseStorageRuntime) Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error) { + return &WriteValueResponse{ + Code: 500, + Message: "unsupportted operation (base)", + }, nil +} + +func (t *baseStorageRuntime) setReadOnly(val bool) *baseStorageRuntime { + t.meta.ReadOnly = val + return t +} + +func (t *baseStorageRuntime) setBuiltin(val bool) *baseStorageRuntime { + t.meta.Builtin = val + return t +} + +type RootStorageMeta struct { + ReadOnly bool `json:"editable,omitempty"` + Builtin bool `json:"builtin,omitempty"` + Ready bool `json:"ready"` // can connect + Notice []data.Notice `json:"notice,omitempty"` + + Config RootStorageConfig `json:"config"` +} diff --git a/pkg/services/store/utils.go b/pkg/services/store/utils.go new file mode 100644 index 00000000000..95777ace5e0 --- /dev/null +++ b/pkg/services/store/utils.go @@ -0,0 +1,31 @@ +package store + +import ( + "path/filepath" + "strings" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/web" +) + +func splitFirstSegment(path string) (string, string) { + idx := strings.Index(path, "/") + if idx == 0 { + path = path[1:] + idx = strings.Index(path, "/") + } + + if idx > 0 { + return path[:idx], path[idx+1:] + } + return path, "" +} + +func getPathAndScope(c *models.ReqContext) (string, string) { + params := web.Params(c.Req) + path := params["*"] + if path == "" { + return "", "" + } + return splitFirstSegment(filepath.Clean(path)) +} diff --git a/pkg/services/store/utils_test.go b/pkg/services/store/utils_test.go new file mode 100644 index 00000000000..c05f4d096f5 --- /dev/null +++ b/pkg/services/store/utils_test.go @@ -0,0 +1,25 @@ +package store + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUtils(t *testing.T) { + a, b := splitFirstSegment("") + require.Equal(t, "", a) + require.Equal(t, "", b) + + a, b = splitFirstSegment("hello") + require.Equal(t, "hello", a) + require.Equal(t, "", b) + + a, b = splitFirstSegment("hello/world") + require.Equal(t, "hello", a) + require.Equal(t, "world", b) + + a, b = splitFirstSegment("/hello/world") // strip leading slash + require.Equal(t, "hello", a) + require.Equal(t, "world", b) +} diff --git a/pkg/tsdb/grafanads/grafana.go b/pkg/tsdb/grafanads/grafana.go index 39cae38f1c8..72e4ccd3d95 100644 --- a/pkg/tsdb/grafanads/grafana.go +++ b/pkg/tsdb/grafanads/grafana.go @@ -1,20 +1,18 @@ package grafanads import ( + "bytes" "context" "encoding/json" "fmt" - "os" "path/filepath" - "strings" "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" - "github.com/grafana/grafana-plugin-sdk-go/experimental" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/searchV2" + "github.com/grafana/grafana/pkg/services/store" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/testdatasource" ) @@ -35,27 +33,18 @@ const DatasourceUID = "grafana" // This is important to do since otherwise we will only get a // not implemented error response from plugin at runtime. var ( - _ backend.QueryDataHandler = (*Service)(nil) - _ backend.CheckHealthHandler = (*Service)(nil) - logger = log.New("tsdb.grafana") + _ backend.QueryDataHandler = (*Service)(nil) + _ backend.CheckHealthHandler = (*Service)(nil) ) -func ProvideService(cfg *setting.Cfg, search searchV2.SearchService) *Service { - return newService(cfg, search) +func ProvideService(cfg *setting.Cfg, search searchV2.SearchService, store store.StorageService) *Service { + return newService(cfg, search, store) } -func newService(cfg *setting.Cfg, search searchV2.SearchService) *Service { +func newService(cfg *setting.Cfg, search searchV2.SearchService, store store.StorageService) *Service { s := &Service{ - staticRootPath: cfg.StaticRootPath, - roots: []string{ - "testdata", - "img/icons", - "img/bg", - "gazetteer", - "maps", - "upload", // does not exist yet - }, search: search, + store: store, } return s @@ -63,10 +52,8 @@ func newService(cfg *setting.Cfg, search searchV2.SearchService) *Service { // Service exists regardless of user settings type Service struct { - // path to the public folder - staticRootPath string - roots []string - search searchV2.SearchService + search searchV2.SearchService + store store.StorageService } func DataSourceModel(orgId int64) *models.DataSource { @@ -89,9 +76,9 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) case queryTypeRandomWalk: response.Responses[q.RefID] = s.doRandomWalk(q) case queryTypeList: - response.Responses[q.RefID] = s.doListQuery(q) + response.Responses[q.RefID] = s.doListQuery(ctx, q) case queryTypeRead: - response.Responses[q.RefID] = s.doReadQuery(q) + response.Responses[q.RefID] = s.doReadQuery(ctx, q) case queryTypeSearch: response.Responses[q.RefID] = s.doSearchQuery(ctx, req, q) default: @@ -111,25 +98,7 @@ func (s *Service) CheckHealth(_ context.Context, _ *backend.CheckHealthRequest) }, nil } -func (s *Service) publicPath(path string) (string, error) { - if strings.Contains(path, "..") { - return "", fmt.Errorf("invalid string") - } - - ok := false - for _, root := range s.roots { - if strings.HasPrefix(path, root) { - ok = true - break - } - } - if !ok { - return "", fmt.Errorf("bad root path") - } - return filepath.Join(s.staticRootPath, path), nil -} - -func (s *Service) doListQuery(query backend.DataQuery) backend.DataResponse { +func (s *Service) doListQuery(ctx context.Context, query backend.DataQuery) backend.DataResponse { q := &listQueryModel{} response := backend.DataResponse{} err := json.Unmarshal(query.JSON, &q) @@ -138,40 +107,17 @@ func (s *Service) doListQuery(query backend.DataQuery) backend.DataResponse { return response } - if q.Path == "" { - count := len(s.roots) - names := data.NewFieldFromFieldType(data.FieldTypeString, count) - mtype := data.NewFieldFromFieldType(data.FieldTypeString, count) - names.Name = "name" - mtype.Name = "mediaType" - for i, f := range s.roots { - names.Set(i, f) - mtype.Set(i, "directory") - } - frame := data.NewFrame("", names, mtype) - frame.SetMeta(&data.FrameMeta{ - Type: data.FrameTypeDirectoryListing, - }) - response.Frames = data.Frames{frame} - } else { - path, err := s.publicPath(q.Path) - if err != nil { - response.Error = err - return response - } - frame, err := experimental.GetDirectoryFrame(path, false) - if err != nil { - response.Error = err - return response - } + path := store.RootPublicStatic + "/" + q.Path + frame, err := s.store.List(ctx, nil, path) + response.Error = err + if frame != nil { response.Frames = data.Frames{frame} } - return response } -func (s *Service) doReadQuery(query backend.DataQuery) backend.DataResponse { - q := &listQueryModel{} +func (s *Service) doReadQuery(ctx context.Context, query backend.DataQuery) backend.DataResponse { + q := &readQueryModel{} response := backend.DataResponse{} err := json.Unmarshal(query.JSON, &q) if err != nil { @@ -184,27 +130,14 @@ func (s *Service) doReadQuery(query backend.DataQuery) backend.DataResponse { return response } - path, err := s.publicPath(q.Path) + path := store.RootPublicStatic + "/" + q.Path + file, err := s.store.Read(ctx, nil, path) if err != nil { response.Error = err return response } - // Can ignore gosec G304 here, because we check the file pattern above - // nolint:gosec - fileReader, err := os.Open(path) - if err != nil { - response.Error = fmt.Errorf("failed to read file") - return response - } - - defer func() { - if err := fileReader.Close(); err != nil { - logger.Warn("Failed to close file", "err", err, "path", path) - } - }() - - frame, err := testdatasource.LoadCsvContent(fileReader, filepath.Base(path)) + frame, err := testdatasource.LoadCsvContent(bytes.NewReader(file.Contents), filepath.Base(path)) if err != nil { response.Error = err return response diff --git a/pkg/tsdb/grafanads/grafana_test.go b/pkg/tsdb/grafanads/grafana_test.go deleted file mode 100644 index d85b7029938..00000000000 --- a/pkg/tsdb/grafanads/grafana_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package grafanads - -import ( - "encoding/json" - "path" - "testing" - - "github.com/grafana/grafana/pkg/services/searchV2" - "github.com/grafana/grafana/pkg/setting" - - "github.com/grafana/grafana-plugin-sdk-go/backend" - "github.com/grafana/grafana-plugin-sdk-go/experimental" - "github.com/stretchr/testify/require" -) - -func asJSON(v interface{}) json.RawMessage { - b, _ := json.Marshal(v) - return b -} - -func TestReadFolderListing(t *testing.T) { - ds := newService(&setting.Cfg{StaticRootPath: "../../../public"}, searchV2.NewStubSearchService()) - dr := ds.doListQuery(backend.DataQuery{ - QueryType: "x", - JSON: asJSON(listQueryModel{ - Path: "testdata", - }), - }) - err := experimental.CheckGoldenDataResponse(path.Join("testdata", "list.golden.txt"), &dr, true) - require.NoError(t, err) -} - -func TestReadCSVFile(t *testing.T) { - ds := newService(&setting.Cfg{StaticRootPath: "../../../public"}, searchV2.NewStubSearchService()) - dr := ds.doReadQuery(backend.DataQuery{ - QueryType: "x", - JSON: asJSON(readQueryModel{ - Path: "testdata/js_libraries.csv", - }), - }) - err := experimental.CheckGoldenDataResponse(path.Join("testdata", "jslib.golden.txt"), &dr, true) - require.NoError(t, err) -} diff --git a/pkg/tsdb/grafanads/testdata/list.golden.txt b/pkg/tsdb/grafanads/testdata/list.golden.txt deleted file mode 100644 index ea6cad48869..00000000000 --- a/pkg/tsdb/grafanads/testdata/list.golden.txt +++ /dev/null @@ -1,25 +0,0 @@ -🌟 This was machine generated. Do not edit. 🌟 - -Frame[0] { - "type": "directory-listing", - "pathSeparator": "/" -} -Name: -Dimensions: 2 Fields by 7 Rows -+--------------------------+------------------+ -| Name: name | Name: media-type | -| Labels: | Labels: | -| Type: []string | Type: []string | -+--------------------------+------------------+ -| browser_marketshare.csv | | -| flight_info_by_state.csv | | -| gdp_per_capita.csv | | -| js_libraries.csv | | -| ohlc_dogecoin.csv | | -| population_by_state.csv | | -| weight_height.csv | | -+--------------------------+------------------+ - - -====== TEST DATA RESPONSE (arrow base64) ====== -FRAME=QVJST1cxAAD/////uAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAKQAAAADAAAATAAAACgAAAAEAAAA0P7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADw/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAABD///8IAAAAPAAAADAAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInBhdGhTZXBhcmF0b3IiOiIvIn0AAAAABAAAAG1ldGEAAAAAAgAAAHwAAAAEAAAAnv///xQAAABAAAAAQAAAAAAAAAU8AAAAAQAAAAQAAACM////CAAAABQAAAAKAAAAbWVkaWEtdHlwZQAABAAAAG5hbWUAAAAAAAAAAIj///8KAAAAbWVkaWEtdHlwZQAAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAG5hbWUAAAAABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAABAAAAG5hbWUAAAAA/////9gAAAAUAAAAAAAAAAwAFgAUABMADAAEAAwAAADQAAAAAAAAABQAAAAAAAADAwAKABgADAAIAAQACgAAABQAAAB4AAAABwAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAACQAAAAAAAAALAAAAAAAAAAAAAAAAAAAACwAAAAAAAAACAAAAAAAAAA0AAAAAAAAAAAAAAAAAAAAAAAAAACAAAABwAAAAAAAAAAAAAAAAAAAAcAAAAAAAAAAAAAAAAAAAAAAAAAFwAAAC8AAABBAAAAUQAAAGIAAAB5AAAAigAAAGJyb3dzZXJfbWFya2V0c2hhcmUuY3N2ZmxpZ2h0X2luZm9fYnlfc3RhdGUuY3N2Z2RwX3Blcl9jYXBpdGEuY3N2anNfbGlicmFyaWVzLmNzdm9obGNfZG9nZWNvaW4uY3N2cG9wdWxhdGlvbl9ieV9zdGF0ZS5jc3Z3ZWlnaHRfaGVpZ2h0LmNzdgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAMAAQAAAMgBAAAAAAAA4AAAAAAAAADQAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAApAAAAAMAAABMAAAAKAAAAAQAAADQ/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAPD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAEP///wgAAAA8AAAAMAAAAHsidHlwZSI6ImRpcmVjdG9yeS1saXN0aW5nIiwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAfAAAAAQAAACe////FAAAAEAAAABAAAAAAAAABTwAAAABAAAABAAAAIz///8IAAAAFAAAAAoAAABtZWRpYS10eXBlAAAEAAAAbmFtZQAAAAAAAAAAiP///woAAABtZWRpYS10eXBlAAAAABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEgAAAAAAAAFRAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAbmFtZQAAAAAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAEAAAAbmFtZQAAAADoAQAAQVJST1cx