Storage: add basic storage service (#46604)

.github/CODEOWNERS

@@ -56,6 +56,7 @@ go.sum @grafana/backend-platform
 # Grafana edge
 /pkg/services/live/ @grafana/grafana-edge-squad
 /pkg/services/searchV2/ @grafana/grafana-edge-squad
+/pkg/services/store/ @grafana/grafana-edge-squad
 /pkg/infra/filestore/ @grafana/grafana-edge-squad
 
 # Alerting

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -46,5 +46,7 @@ export interface FeatureToggles {
   dashboardComments?: boolean;
   annotationComments?: boolean;
   migrationLocking?: boolean;
+  storage?: boolean;
+  storageLocalUpload?: boolean;
   azureMonitorResourcePickerForMetrics?: boolean;
 }

pkg/api/api.go

@@ -211,6 +211,17 @@ func (hs *HTTPServer) registerRoutes() {
 			orgRoute.Get("/quotas", authorize(reqSignedIn, ac.EvalPermission(ActionOrgsQuotasRead)), routing.Wrap(hs.GetCurrentOrgQuotas))
 		})
 
+		if hs.Features.IsEnabled(featuremgmt.FlagStorage) {
+			apiRoute.Group("/storage", func(orgRoute routing.RouteRegister) {
+				orgRoute.Get("/list/", routing.Wrap(hs.StorageService.List))
+				orgRoute.Get("/list/*", routing.Wrap(hs.StorageService.List))
+				orgRoute.Get("/read/*", routing.Wrap(hs.StorageService.Read))
+
+				orgRoute.Delete("/delete/*", reqSignedIn, routing.Wrap(hs.StorageService.Delete))
+				orgRoute.Post("/upload", reqSignedIn, routing.Wrap(hs.StorageService.Upload))
+			})
+		}
+
 		// current org
 		apiRoute.Group("/org", func(orgRoute routing.RouteRegister) {
 			userIDScope := ac.Scope("users", "id", ac.Parameter(":userId"))

pkg/api/http_server.go

@@ -62,6 +62,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/serviceaccounts"
 	"github.com/grafana/grafana/pkg/services/shorturls"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/services/store"
 	"github.com/grafana/grafana/pkg/services/teamguardian"
 	"github.com/grafana/grafana/pkg/services/thumbs"
 	"github.com/grafana/grafana/pkg/services/updatechecker"
@@ -110,6 +111,7 @@ type HTTPServer struct {
 	Live                         *live.GrafanaLive
 	LivePushGateway              *pushhttp.Gateway
 	ThumbService                 thumbs.Service
+	StorageService               store.HTTPStorageService
 	ContextHandler               *contexthandler.ContextHandler
 	SQLStore                     sqlstore.Store
 	AlertEngine                  *alerting.AlertEngine
@@ -169,7 +171,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 	pluginsUpdateChecker *updatechecker.PluginsService, searchUsersService searchusers.Service,
 	dataSourcesService datasources.DataSourceService, secretsService secrets.Service, queryDataService *query.Service,
 	ldapGroups ldap.Groups, teamGuardian teamguardian.TeamGuardian, serviceaccountsService serviceaccounts.Service,
-	authInfoService login.AuthInfoService, permissionsServices accesscontrol.PermissionsServices,
+	authInfoService login.AuthInfoService, permissionsServices accesscontrol.PermissionsServices, storageService store.HTTPStorageService,
 	notificationService *notifications.NotificationService, dashboardService dashboards.DashboardService,
 	dashboardProvisioningService dashboards.DashboardProvisioningService, folderService dashboards.FolderService,
 	datasourcePermissionsService permissions.DatasourcePermissionsService, alertNotificationService *alerting.AlertNotificationService,
@@ -204,6 +206,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 		QueryHistoryService:          queryHistoryService,
 		Features:                     features,
 		ThumbService:                 thumbService,
+		StorageService:               storageService,
 		RemoteCacheService:           remoteCache,
 		ProvisioningService:          provisioningService,
 		Login:                        loginService,

pkg/infra/filestorage/cdk_blob_filestorage.go

@@ -391,7 +391,7 @@ func (c cdkBlobStorage) list(ctx context.Context, folderPath string, paging *Pag
 	hasMore := false
 	if len(files) > pageSize {
 		hasMore = true
-		files = files[:len(files)-pageSize]
+		files = files[:pageSize]
 	}
 
 	lastPath := ""

pkg/infra/filestorage/fs_integration_test.go

@@ -427,17 +427,19 @@ func TestFsStorage(t *testing.T) {
 						},
 					},
 					queryListFiles{
-						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: ""}},
-						list:  checks(listSize(1), listHasMore(true), listLastPath("/folder1/a")),
+						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 2, After: ""}},
+						list:  checks(listSize(2), listHasMore(true), listLastPath("/folder1/b")),
 						files: [][]interface{}{
 							checks(fPath("/folder1/a")),
+							checks(fPath("/folder1/b")),
 						},
 					},
 					queryListFiles{
-						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 1, After: ""}},
-						list:  checks(listSize(1), listHasMore(true), listLastPath("/folder1")),
+						input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true, WithFiles: true, WithFolders: true}, paging: &Paging{First: 2, After: ""}},
+						list:  checks(listSize(2), listHasMore(true), listLastPath("/folder1/a")),
 						files: [][]interface{}{
 							checks(fPath("/folder1"), fMimeType(DirectoryMimeType)),
+							checks(fPath("/folder1/a")),
 						},
 					},
 					queryListFiles{

pkg/plugins/manager/manager_integration_test.go

@@ -87,7 +87,7 @@ func TestPluginManager_int_init(t *testing.T) {
 	my := mysql.ProvideService(cfg, hcp)
 	ms := mssql.ProvideService(cfg)
 	sv2 := searchV2.ProvideService(sqlstore.InitTestDB(t))
-	graf := grafanads.ProvideService(cfg, sv2)
+	graf := grafanads.ProvideService(cfg, sv2, nil)
 
 	coreRegistry := coreplugin.ProvideCoreRegistry(am, cw, cm, es, grap, idb, lk, otsdb, pr, tmpo, td, pg, my, ms, graf)
 

pkg/server/backgroundsvcs/background_services.go

@@ -22,6 +22,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/rendering"
 	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
 	"github.com/grafana/grafana/pkg/services/serviceaccounts"
+	"github.com/grafana/grafana/pkg/services/store"
 	"github.com/grafana/grafana/pkg/services/thumbs"
 	"github.com/grafana/grafana/pkg/services/updatechecker"
 )
@@ -33,7 +34,7 @@ func ProvideBackgroundServiceRegistry(
 	provisioning *provisioning.ProvisioningServiceImpl, alerting *alerting.AlertEngine, usageStats *uss.UsageStats,
 	grafanaUpdateChecker *updatechecker.GrafanaService, pluginsUpdateChecker *updatechecker.PluginsService,
 	metrics *metrics.InternalMetricsService, secretsService *secretsManager.SecretsService,
-	remoteCache *remotecache.RemoteCache, thumbnailsService thumbs.Service,
+	remoteCache *remotecache.RemoteCache, thumbnailsService thumbs.Service, StorageService store.StorageService,
 	// Need to make sure these are initialized, is there a better place to put them?
 	_ *dashboardsnapshots.Service, _ *alerting.AlertNotificationService,
 	_ serviceaccounts.Service, _ *guardian.Provider,
@@ -58,6 +59,7 @@ func ProvideBackgroundServiceRegistry(
 		tracing,
 		remoteCache,
 		secretsService,
+		StorageService,
 		thumbnailsService)
 }
 

pkg/server/wire.go

@@ -75,6 +75,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/shorturls"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/services/sqlstore/mockstore"
+	"github.com/grafana/grafana/pkg/services/store"
 	"github.com/grafana/grafana/pkg/services/teamguardian"
 	teamguardianDatabase "github.com/grafana/grafana/pkg/services/teamguardian/database"
 	teamguardianManager "github.com/grafana/grafana/pkg/services/teamguardian/manager"
@@ -158,6 +159,8 @@ var wireBasicSet = wire.NewSet(
 	datasourceproxy.ProvideService,
 	search.ProvideService,
 	searchV2.ProvideService,
+	store.ProvideService,
+	store.ProvideHTTPService,
 	live.ProvideService,
 	pushhttp.ProvideService,
 	plugincontext.ProvideService,

pkg/services/featuremgmt/registry.go

@@ -162,6 +162,17 @@ var (
 			Description: "Lock database during migrations",
 			State:       FeatureStateBeta,
 		},
+		{
+			Name:        "storage",
+			Description: "Configurable storage for dashboards, datasources, and resources",
+			State:       FeatureStateAlpha,
+		},
+		{
+			Name:            "storageLocalUpload",
+			Description:     "allow uploads to local storage",
+			State:           FeatureStateAlpha,
+			RequiresDevMode: true,
+		},
 		{
 			Name:            "azureMonitorResourcePickerForMetrics",
 			Description:     "New UI for Azure Monitor Metrics Query",

pkg/services/featuremgmt/toggles_gen.go

@@ -123,6 +123,14 @@ const (
 	// Lock database during migrations
 	FlagMigrationLocking = "migrationLocking"
 
+	// FlagStorage
+	// Configurable storage for dashboards, datasources, and resources
+	FlagStorage = "storage"
+
+	// FlagStorageLocalUpload
+	// allow uploads to local storage
+	FlagStorageLocalUpload = "storageLocalUpload"
+
 	// FlagAzureMonitorResourcePickerForMetrics
 	// New UI for Azure Monitor Metrics Query
 	FlagAzureMonitorResourcePickerForMetrics = "azureMonitorResourcePickerForMetrics"

pkg/services/store/config.go

@@ -0,0 +1,53 @@
+package store
+
+type RootStorageConfig struct {
+	Type   string `json:"type"`
+	Prefix string `json:"prefix"`
+	Name   string `json:"name"`
+
+	// Depending on type, these will be configured
+	Disk *StorageLocalDiskConfig `json:"disk,omitempty"`
+	Git  *StorageGitConfig       `json:"git,omitempty"`
+	SQL  *StorageSQLConfig       `json:"sql,omitempty"`
+	S3   *StorageS3Config        `json:"s3,omitempty"`
+	GCS  *StorageGCSConfig       `json:"gcs,omitempty"`
+}
+
+type StorageLocalDiskConfig struct {
+	Path  string   `json:"path"`
+	Roots []string `json:"roots,omitempty"` // null is everything
+}
+
+type StorageGitConfig struct {
+	Remote string `json:"remote"`
+	Branch string `json:"branch"`
+	Root   string `json:"root"` // subfolder within the remote
+
+	// Pull interval?
+	// Requires pull request?
+	RequirePullRequest bool `json:"requirePullRequest"`
+
+	// SECURE JSON :grimicing:
+	AccessToken string `json:"accessToken,omitempty"` // Simplest auth method for github
+}
+
+type StorageSQLConfig struct {
+	// no custom settings
+}
+
+type StorageS3Config struct {
+	Bucket string `json:"bucket"`
+	Folder string `json:"folder"`
+
+	// SECURE!!!
+	AccessKey string `json:"accessKey"`
+	SecretKey string `json:"secretKey"`
+	Region    string `json:"region"`
+}
+
+type StorageGCSConfig struct {
+	Bucket string `json:"bucket"`
+	Folder string `json:"folder"`
+
+	CredentialsFile string `json:"credentialsFile"`
+}

pkg/services/store/http.go

@@ -0,0 +1,71 @@
+package store
+
+import (
+	"github.com/grafana/grafana/pkg/api/response"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/web"
+)
+
+// HTTPStorageService passes raw HTTP requests to a well typed storage service
+type HTTPStorageService interface {
+	List(c *models.ReqContext) response.Response
+	Read(c *models.ReqContext) response.Response
+	Delete(c *models.ReqContext) response.Response
+	Upload(c *models.ReqContext) response.Response
+}
+
+type httpStorage struct {
+	store StorageService
+}
+
+func ProvideHTTPService(store StorageService) HTTPStorageService {
+	return &httpStorage{
+		store: store,
+	}
+}
+
+func (s *httpStorage) Upload(c *models.ReqContext) response.Response {
+	action := "Upload"
+	scope, path := getPathAndScope(c)
+
+	return response.JSON(200, map[string]string{
+		"action": action,
+		"scope":  scope,
+		"path":   path,
+	})
+}
+
+func (s *httpStorage) Read(c *models.ReqContext) response.Response {
+	action := "Read"
+	scope, path := getPathAndScope(c)
+
+	return response.JSON(200, map[string]string{
+		"action": action,
+		"scope":  scope,
+		"path":   path,
+	})
+}
+
+func (s *httpStorage) Delete(c *models.ReqContext) response.Response {
+	action := "Delete"
+	scope, path := getPathAndScope(c)
+
+	return response.JSON(200, map[string]string{
+		"action": action,
+		"scope":  scope,
+		"path":   path,
+	})
+}
+
+func (s *httpStorage) List(c *models.ReqContext) response.Response {
+	params := web.Params(c.Req)
+	path := params["*"]
+	frame, err := s.store.List(c.Req.Context(), c.SignedInUser, path)
+	if err != nil {
+		return response.Error(400, "error reading path", err)
+	}
+	if frame == nil {
+		return response.Error(404, "not found", nil)
+	}
+	return response.JSONStreaming(200, frame)
+}

pkg/services/store/service.go

@@ -0,0 +1,88 @@
+package store
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/infra/filestorage"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/registry"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+var grafanaStorageLogger = log.New("grafanaStorageLogger")
+
+const RootPublicStatic = "public-static"
+
+type StorageService interface {
+	registry.BackgroundService
+
+	// List folder contents
+	List(ctx context.Context, user *models.SignedInUser, path string) (*data.Frame, error)
+
+	// Read raw file contents out of the store
+	Read(ctx context.Context, user *models.SignedInUser, path string) (*filestorage.File, error)
+}
+
+type standardStorageService struct {
+	sql  *sqlstore.SQLStore
+	tree *nestedTree
+}
+
+func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, cfg *setting.Cfg) StorageService {
+	roots := []storageRuntime{
+		newDiskStorage(RootPublicStatic, "Public static files", &StorageLocalDiskConfig{
+			Path: cfg.StaticRootPath,
+			Roots: []string{
+				"/testdata/",
+				// "/img/icons/",
+				// "/img/bg/",
+				"/img/",
+				"/gazetteer/",
+				"/maps/",
+			},
+		}).setReadOnly(true).setBuiltin(true),
+	}
+
+	storage := filepath.Join(cfg.DataPath, "storage")
+	_ = os.MkdirAll(storage, 0700)
+
+	if features.IsEnabled(featuremgmt.FlagStorageLocalUpload) {
+		roots = append(roots, newDiskStorage("upload", "Local file upload", &StorageLocalDiskConfig{
+			Path: filepath.Join(storage, "upload"),
+		}))
+	}
+	s := newStandardStorageService(roots)
+	s.sql = sql
+	return s
+}
+
+func newStandardStorageService(roots []storageRuntime) *standardStorageService {
+	res := &nestedTree{
+		roots: roots,
+	}
+	res.init()
+	return &standardStorageService{
+		tree: res,
+	}
+}
+
+func (s *standardStorageService) Run(ctx context.Context) error {
+	grafanaStorageLogger.Info("storage starting")
+	return nil
+}
+
+func (s *standardStorageService) List(ctx context.Context, user *models.SignedInUser, path string) (*data.Frame, error) {
+	// apply access control here
+	return s.tree.ListFolder(ctx, path)
+}
+
+func (s *standardStorageService) Read(ctx context.Context, user *models.SignedInUser, path string) (*filestorage.File, error) {
+	// TODO: permission check!
+	return s.tree.GetFile(ctx, path)
+}

pkg/services/store/service_test.go

@@ -0,0 +1,47 @@
+package store
+
+import (
+	"bytes"
+	"context"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/experimental"
+	"github.com/grafana/grafana/pkg/tsdb/testdatasource"
+	"github.com/stretchr/testify/require"
+)
+
+func TestListFiles(t *testing.T) {
+	publicRoot, err := filepath.Abs("../../../public")
+	require.NoError(t, err)
+	roots := []storageRuntime{
+		newDiskStorage("public", "Public static files", &StorageLocalDiskConfig{
+			Path: publicRoot,
+			Roots: []string{
+				"/testdata/",
+				"/img/icons/",
+				"/img/bg/",
+				"/gazetteer/",
+				"/maps/",
+				"/upload/",
+			},
+		}).setReadOnly(true).setBuiltin(true),
+	}
+
+	store := newStandardStorageService(roots)
+	frame, err := store.List(context.Background(), nil, "public/testdata")
+	require.NoError(t, err)
+
+	err = experimental.CheckGoldenFrame(path.Join("testdata", "public_testdata.golden.txt"), frame, true)
+	require.NoError(t, err)
+
+	file, err := store.Read(context.Background(), nil, "public/testdata/js_libraries.csv")
+	require.NoError(t, err)
+	require.NotNil(t, file)
+
+	frame, err = testdatasource.LoadCsvContent(bytes.NewReader(file.Contents), file.Name)
+	require.NoError(t, err)
+	err = experimental.CheckGoldenFrame(path.Join("testdata", "public_testdata_js_libraries.golden.txt"), frame, true)
+	require.NoError(t, err)
+}

pkg/services/store/storage_disk.go

@@ -0,0 +1,91 @@
+package store
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/infra/filestorage"
+	"gocloud.dev/blob"
+)
+
+const rootStorageTypeDisk = "disk"
+
+type rootStorageDisk struct {
+	baseStorageRuntime
+
+	settings *StorageLocalDiskConfig
+}
+
+func newDiskStorage(prefix string, name string, cfg *StorageLocalDiskConfig) *rootStorageDisk {
+	if cfg == nil {
+		cfg = &StorageLocalDiskConfig{}
+	}
+
+	meta := RootStorageMeta{
+		Config: RootStorageConfig{
+			Type:   rootStorageTypeDisk,
+			Prefix: prefix,
+			Name:   name,
+			Disk:   cfg,
+		},
+	}
+	if prefix == "" {
+		meta.Notice = append(meta.Notice, data.Notice{
+			Severity: data.NoticeSeverityError,
+			Text:     "Missing prefix",
+		})
+	}
+	if cfg.Path == "" {
+		meta.Notice = append(meta.Notice, data.Notice{
+			Severity: data.NoticeSeverityError,
+			Text:     "Missing path configuration",
+		})
+	}
+	s := &rootStorageDisk{}
+
+	if meta.Notice == nil {
+		path := fmt.Sprintf("file://%s", cfg.Path)
+		bucket, err := blob.OpenBucket(context.Background(), path)
+		if err != nil {
+			grafanaStorageLogger.Warn("error loading storage", "prefix", prefix, "err", err)
+			meta.Notice = append(meta.Notice, data.Notice{
+				Severity: data.NoticeSeverityError,
+				Text:     "Failed to initialize storage",
+			})
+		} else {
+			s.store = filestorage.NewCdkBlobStorage(grafanaStorageLogger,
+				bucket, "",
+				filestorage.NewPathFilters(cfg.Roots, nil, nil, nil))
+
+			meta.Ready = true // exists!
+		}
+	}
+
+	s.meta = meta
+	s.settings = cfg
+	return s
+}
+
+func (s *rootStorageDisk) Sync() error {
+	return nil // already in sync
+}
+
+// with local disk user metadata and messages are lost
+func (s *rootStorageDisk) Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error) {
+	byteAray := []byte(cmd.Body)
+
+	path := cmd.Path
+	if !strings.HasPrefix(path, filestorage.Delimiter) {
+		path = filestorage.Delimiter + path
+	}
+	err := s.store.Upsert(ctx, &filestorage.UpsertFileCommand{
+		Path:     path,
+		Contents: &byteAray,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &WriteValueResponse{Code: 200}, nil
+}

pkg/services/store/testdata/public_testdata.golden.txt

@@ -0,0 +1,27 @@
+🌟 This was machine generated.  Do not edit. 🌟
+
+Frame[0] {
+    "type": "directory-listing",
+    "custom": {
+        "HasMore": false
+    }
+}
+Name: 
+Dimensions: 3 Fields by 7 Rows
++--------------------------+-------------------------+---------------+
+| Name: name               | Name: mediaType         | Name: size    |
+| Labels:                  | Labels:                 | Labels:       |
+| Type: []string           | Type: []string          | Type: []int64 |
++--------------------------+-------------------------+---------------+
+| browser_marketshare.csv  | text/csv; charset=utf-8 | 355           |
+| flight_info_by_state.csv | text/csv; charset=utf-8 | 681           |
+| gdp_per_capita.csv       | text/csv; charset=utf-8 | 4116          |
+| js_libraries.csv         | text/csv; charset=utf-8 | 179           |
+| ohlc_dogecoin.csv        | text/csv; charset=utf-8 | 191804        |
+| population_by_state.csv  | text/csv; charset=utf-8 | 138           |
+| weight_height.csv        | text/csv; charset=utf-8 | 418121        |
++--------------------------+-------------------------+---------------+
+
+
+====== TEST DATA RESPONSE (arrow base64) ======
+FRAME=QVJST1cxAAD/////WAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAKgAAAADAAAATAAAACgAAAAEAAAAMP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABQ/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAHD+//8IAAAAQAAAADcAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsImN1c3RvbSI6eyJIYXNNb3JlIjpmYWxzZX19AAQAAABtZXRhAAAAAAMAAAAYAQAApAAAAAQAAAAG////FAAAAHAAAAB4AAAAAAAAAnwAAAACAAAALAAAAAQAAAD4/v//CAAAABAAAAAEAAAAc2l6ZQAAAAAEAAAAbmFtZQAAAAAc////CAAAABwAAAAQAAAAeyJ1bml0IjoiYnl0ZXMifQAAAAAGAAAAY29uZmlnAAAAAAAACAAMAAgABwAIAAAAAAAAAUAAAAAEAAAAc2l6ZQAAAACi////FAAAAEAAAABAAAAAAAAABTwAAAABAAAABAAAAJD///8IAAAAFAAAAAkAAABtZWRpYVR5cGUAAAAEAAAAbmFtZQAAAAAAAAAAjP///wkAAABtZWRpYVR5cGUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABuYW1lAAAAAAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP////8IAQAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAsAEAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAmAAAAAcAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAigAAAAAAAACwAAAAAAAAAAAAAAAAAAAAsAAAAAAAAAAgAAAAAAAAANAAAAAAAAAAoQAAAAAAAAB4AQAAAAAAAAAAAAAAAAAAeAEAAAAAAAA4AAAAAAAAAAAAAAADAAAABwAAAAAAAAAAAAAAAAAAAAcAAAAAAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAABcAAAAvAAAAQQAAAFEAAABiAAAAeQAAAIoAAABicm93c2VyX21hcmtldHNoYXJlLmNzdmZsaWdodF9pbmZvX2J5X3N0YXRlLmNzdmdkcF9wZXJfY2FwaXRhLmNzdmpzX2xpYnJhcmllcy5jc3ZvaGxjX2RvZ2Vjb2luLmNzdnBvcHVsYXRpb25fYnlfc3RhdGUuY3N2d2VpZ2h0X2hlaWdodC5jc3YAAAAAAAAAAAAAFwAAAC4AAABFAAAAXAAAAHMAAACKAAAAoQAAAHRleHQvY3N2OyBjaGFyc2V0PXV0Zi04dGV4dC9jc3Y7IGNoYXJzZXQ9dXRmLTh0ZXh0L2NzdjsgY2hhcnNldD11dGYtOHRleHQvY3N2OyBjaGFyc2V0PXV0Zi04dGV4dC9jc3Y7IGNoYXJzZXQ9dXRmLTh0ZXh0L2NzdjsgY2hhcnNldD11dGYtOHRleHQvY3N2OyBjaGFyc2V0PXV0Zi04AAAAAAAAAGMBAAAAAAAAqQIAAAAAAAAUEAAAAAAAALMAAAAAAAAAPO0CAAAAAACKAAAAAAAAAElhBgAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAQAAQAAAGgCAAAAAAAAEAEAAAAAAACwAQAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAqAAAAAMAAABMAAAAKAAAAAQAAAAw/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAFD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAcP7//wgAAABAAAAANwAAAHsidHlwZSI6ImRpcmVjdG9yeS1saXN0aW5nIiwiY3VzdG9tIjp7Ikhhc01vcmUiOmZhbHNlfX0ABAAAAG1ldGEAAAAAAwAAABgBAACkAAAABAAAAAb///8UAAAAcAAAAHgAAAAAAAACfAAAAAIAAAAsAAAABAAAAPj+//8IAAAAEAAAAAQAAABzaXplAAAAAAQAAABuYW1lAAAAABz///8IAAAAHAAAABAAAAB7InVuaXQiOiJieXRlcyJ9AAAAAAYAAABjb25maWcAAAAAAAAIAAwACAAHAAgAAAAAAAABQAAAAAQAAABzaXplAAAAAKL///8UAAAAQAAAAEAAAAAAAAAFPAAAAAEAAAAEAAAAkP///wgAAAAUAAAACQAAAG1lZGlhVHlwZQAAAAQAAABuYW1lAAAAAAAAAACM////CQAAAG1lZGlhVHlwZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAG5hbWUAAAAABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAABAAAAG5hbWUAAAAAiAIAAEFSUk9XMQ==

pkg/services/store/testdata/public_testdata_js_libraries.golden.txt

@@ -18,4 +18,4 @@ Dimensions: 4 Fields by 6 Rows
 
 
 ====== TEST DATA RESPONSE (arrow base64) ======
-FRAME=QVJST1cxAAD/////WAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAGAAAAACAAAAKAAAAAQAAAAs/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAEz+//8IAAAAHAAAABAAAABqc19saWJyYXJpZXMuY3N2AAAAAAQAAABuYW1lAAAAAAQAAABgAQAA1AAAAHAAAAAEAAAAwv7//xQAAABAAAAAQAAAAAAAAgFEAAAAAQAAAAQAAACw/v//CAAAABQAAAAIAAAAV2F0Y2hlcnMAAAAABAAAAG5hbWUAAAAAAAAAADT///8AAAABQAAAAAgAAABXYXRjaGVycwAAAAAq////FAAAADwAAAA8AAAAAAACAUAAAAABAAAABAAAABj///8IAAAAEAAAAAUAAABGb3JrcwAAAAQAAABuYW1lAAAAAAAAAACY////AAAAAUAAAAAFAAAARm9ya3MAAACK////FAAAAEQAAABMAAAAAAACAVAAAAABAAAABAAAAHj///8IAAAAGAAAAAwAAABHaXRodWIgU3RhcnMAAAAABAAAAG5hbWUAAAAAAAAAAAgADAAIAAcACAAAAAAAAAFAAAAADAAAAEdpdGh1YiBTdGFycwAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAABQFEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAcAAABMaWJyYXJ5AAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAcAAABMaWJyYXJ5AP////8oAQAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAA2AAAAAAAAAAUAAAAAAAAAwMACgAYAAwACAAEAAoAAAAUAAAAqAAAAAYAAAAAAAAAAAAAAAkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAKAAAAAAAAABIAAAAAAAAAAAAAAAAAAAASAAAAAAAAAAwAAAAAAAAAHgAAAAAAAAAAAAAAAAAAAB4AAAAAAAAADAAAAAAAAAAqAAAAAAAAAAAAAAAAAAAAKgAAAAAAAAAMAAAAAAAAAAAAAAABAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAsAAAASAAAAGAAAAB4AAAAlAAAAAAAAAFJlYWN0LmpzVnVlQW5ndWxhckpRdWVyeU1ldGVvckF1cmVsaWEAAAAolAIAAAAAAMDOAgAAAAAAuB4BAAAAAAB01gAAAAAAAKClAAAAAAAAUC0AAAAAAADQhAAAAAAAAKxxAAAAAAAAZEsAAAAAAAAgTgAAAAAAAFAUAAAAAAAArAIAAAAAAAAsGgAAAAAAAJwYAAAAAAAAgAwAAAAAAADkDAAAAAAAAKQGAAAAAAAAugEAAAAAAAAQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAAAwABAAAAaAIAAAAAAAAwAQAAAAAAANgAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABgAAAAAgAAACgAAAAEAAAALP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABM/v//CAAAABwAAAAQAAAAanNfbGlicmFyaWVzLmNzdgAAAAAEAAAAbmFtZQAAAAAEAAAAYAEAANQAAABwAAAABAAAAML+//8UAAAAQAAAAEAAAAAAAAIBRAAAAAEAAAAEAAAAsP7//wgAAAAUAAAACAAAAFdhdGNoZXJzAAAAAAQAAABuYW1lAAAAAAAAAAA0////AAAAAUAAAAAIAAAAV2F0Y2hlcnMAAAAAKv///xQAAAA8AAAAPAAAAAAAAgFAAAAAAQAAAAQAAAAY////CAAAABAAAAAFAAAARm9ya3MAAAAEAAAAbmFtZQAAAAAAAAAAmP///wAAAAFAAAAABQAAAEZvcmtzAAAAiv///xQAAABEAAAATAAAAAAAAgFQAAAAAQAAAAQAAAB4////CAAAABgAAAAMAAAAR2l0aHViIFN0YXJzAAAAAAQAAABuYW1lAAAAAAAAAAAIAAwACAAHAAgAAAAAAAABQAAAAAwAAABHaXRodWIgU3RhcnMAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAARAAAAEgAAAAAAAUBRAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAHAAAATGlicmFyeQAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAHAAAATGlicmFyeQCIAgAAQVJST1cx
+FRAME=QVJST1cxAAD/////WAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAGAAAAACAAAAKAAAAAQAAAAs/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAEz+//8IAAAAHAAAABAAAABqc19saWJyYXJpZXMuY3N2AAAAAAQAAABuYW1lAAAAAAQAAABgAQAA1AAAAHAAAAAEAAAAwv7//xQAAABAAAAAQAAAAAAAAgFEAAAAAQAAAAQAAACw/v//CAAAABQAAAAIAAAAV2F0Y2hlcnMAAAAABAAAAG5hbWUAAAAAAAAAADT///8AAAABQAAAAAgAAABXYXRjaGVycwAAAAAq////FAAAADwAAAA8AAAAAAACAUAAAAABAAAABAAAABj///8IAAAAEAAAAAUAAABGb3JrcwAAAAQAAABuYW1lAAAAAAAAAACY////AAAAAUAAAAAFAAAARm9ya3MAAACK////FAAAAEQAAABMAAAAAAACAVAAAAABAAAABAAAAHj///8IAAAAGAAAAAwAAABHaXRodWIgU3RhcnMAAAAABAAAAG5hbWUAAAAAAAAAAAgADAAIAAcACAAAAAAAAAFAAAAADAAAAEdpdGh1YiBTdGFycwAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAABQFEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAcAAABMaWJyYXJ5AAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAcAAABMaWJyYXJ5AP////8oAQAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAA2AAAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAqAAAAAYAAAAAAAAAAAAAAAkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAcAAAAAAAAACAAAAAAAAAAJQAAAAAAAABIAAAAAAAAAAAAAAAAAAAASAAAAAAAAAAwAAAAAAAAAHgAAAAAAAAAAAAAAAAAAAB4AAAAAAAAADAAAAAAAAAAqAAAAAAAAAAAAAAAAAAAAKgAAAAAAAAAMAAAAAAAAAAAAAAABAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAsAAAASAAAAGAAAAB4AAAAlAAAAAAAAAFJlYWN0LmpzVnVlQW5ndWxhckpRdWVyeU1ldGVvckF1cmVsaWEAAAAolAIAAAAAAMDOAgAAAAAAuB4BAAAAAAB01gAAAAAAAKClAAAAAAAAUC0AAAAAAADQhAAAAAAAAKxxAAAAAAAAZEsAAAAAAAAgTgAAAAAAAFAUAAAAAAAArAIAAAAAAAAsGgAAAAAAAJwYAAAAAAAAgAwAAAAAAADkDAAAAAAAAKQGAAAAAAAAugEAAAAAAAAQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAABAABAAAAaAIAAAAAAAAwAQAAAAAAANgAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABgAAAAAgAAACgAAAAEAAAALP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABM/v//CAAAABwAAAAQAAAAanNfbGlicmFyaWVzLmNzdgAAAAAEAAAAbmFtZQAAAAAEAAAAYAEAANQAAABwAAAABAAAAML+//8UAAAAQAAAAEAAAAAAAAIBRAAAAAEAAAAEAAAAsP7//wgAAAAUAAAACAAAAFdhdGNoZXJzAAAAAAQAAABuYW1lAAAAAAAAAAA0////AAAAAUAAAAAIAAAAV2F0Y2hlcnMAAAAAKv///xQAAAA8AAAAPAAAAAAAAgFAAAAAAQAAAAQAAAAY////CAAAABAAAAAFAAAARm9ya3MAAAAEAAAAbmFtZQAAAAAAAAAAmP///wAAAAFAAAAABQAAAEZvcmtzAAAAiv///xQAAABEAAAATAAAAAAAAgFQAAAAAQAAAAQAAAB4////CAAAABgAAAAMAAAAR2l0aHViIFN0YXJzAAAAAAQAAABuYW1lAAAAAAAAAAAIAAwACAAHAAgAAAAAAAABQAAAAAwAAABHaXRodWIgU3RhcnMAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAARAAAAEgAAAAAAAUBRAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAHAAAATGlicmFyeQAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAHAAAATGlicmFyeQCIAgAAQVJST1cx

pkg/services/store/tree.go

@@ -0,0 +1,106 @@
+package store
+
+import (
+	"context"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/infra/filestorage"
+)
+
+type nestedTree struct {
+	roots  []storageRuntime
+	lookup map[string]filestorage.FileStorage
+}
+
+var (
+	_ storageTree = (*nestedTree)(nil)
+)
+
+func (t *nestedTree) init() {
+	t.lookup = make(map[string]filestorage.FileStorage, len(t.roots))
+	for _, root := range t.roots {
+		t.lookup[root.Meta().Config.Prefix] = root.Store()
+	}
+}
+
+func (t *nestedTree) getRoot(path string) (filestorage.FileStorage, string) {
+	if path == "" {
+		return nil, ""
+	}
+
+	rootKey, path := splitFirstSegment(path)
+	root, ok := t.lookup[rootKey]
+	if !ok || root == nil {
+		return nil, path // not found or not ready
+	}
+	return root, filestorage.Delimiter + path
+}
+
+func (t *nestedTree) GetFile(ctx context.Context, path string) (*filestorage.File, error) {
+	if path == "" {
+		return nil, nil // not found
+	}
+
+	root, path := t.getRoot(path)
+	if root == nil {
+		return nil, nil // not found (or not ready)
+	}
+	return root.Get(ctx, path)
+}
+
+func (t *nestedTree) ListFolder(ctx context.Context, path string) (*data.Frame, error) {
+	if path == "" || path == "/" {
+		count := len(t.roots)
+		title := data.NewFieldFromFieldType(data.FieldTypeString, count)
+		names := data.NewFieldFromFieldType(data.FieldTypeString, count)
+		mtype := data.NewFieldFromFieldType(data.FieldTypeString, count)
+		title.Name = "title"
+		names.Name = "name"
+		mtype.Name = "mediaType"
+		for i, f := range t.roots {
+			names.Set(i, f.Meta().Config.Prefix)
+			title.Set(i, f.Meta().Config.Name)
+			mtype.Set(i, "directory")
+		}
+		frame := data.NewFrame("", names, title, mtype)
+		frame.SetMeta(&data.FrameMeta{
+			Type: data.FrameTypeDirectoryListing,
+		})
+		return frame, nil
+	}
+
+	root, path := t.getRoot(path)
+	if root == nil {
+		return nil, nil // not found (or not ready)
+	}
+
+	listResponse, err := root.List(ctx, path, nil, &filestorage.ListOptions{Recursive: false, WithFolders: true, WithFiles: true})
+
+	if err != nil {
+		return nil, err
+	}
+
+	count := len(listResponse.Files)
+	names := data.NewFieldFromFieldType(data.FieldTypeString, count)
+	mtype := data.NewFieldFromFieldType(data.FieldTypeString, count)
+	fsize := data.NewFieldFromFieldType(data.FieldTypeInt64, count)
+	names.Name = "name"
+	mtype.Name = "mediaType"
+	fsize.Name = "size"
+	fsize.Config = &data.FieldConfig{
+		Unit: "bytes",
+	}
+	for i, f := range listResponse.Files {
+		names.Set(i, f.Name)
+		mtype.Set(i, f.MimeType)
+		fsize.Set(i, f.Size)
+	}
+	frame := data.NewFrame("", names, mtype, fsize)
+	frame.SetMeta(&data.FrameMeta{
+		Type: data.FrameTypeDirectoryListing,
+		Custom: map[string]interface{}{
+			"HasMore": listResponse.HasMore,
+		},
+	})
+	return frame, nil
+}

pkg/services/store/types.go

@@ -0,0 +1,92 @@
+package store
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/infra/filestorage"
+	"github.com/grafana/grafana/pkg/models"
+)
+
+type WriteValueRequest struct {
+	Path    string
+	User    *models.SignedInUser
+	Body    json.RawMessage `json:"body,omitempty"`
+	Message string          `json:"message,omitempty"`
+	Title   string          `json:"title,omitempty"`  // For PRs
+	Action  string          `json:"action,omitempty"` // pr | save
+}
+
+type WriteValueResponse struct {
+	Code    int    `json:"code,omitempty"`
+	Message string `json:"message,omitempty"`
+	URL     string `json:"url,omitempty"`
+	Hash    string `json:"hash,omitempty"`
+	Branch  string `json:"branch,omitempty"`
+	Pending bool   `json:"pending,omitempty"`
+	Size    int64  `json:"size,omitempty"`
+}
+
+type storageTree interface {
+	GetFile(ctx context.Context, path string) (*filestorage.File, error)
+	ListFolder(ctx context.Context, path string) (*data.Frame, error)
+}
+
+//-------------------------------------------
+// INTERNAL
+//-------------------------------------------
+
+type storageRuntime interface {
+	Meta() RootStorageMeta
+
+	Store() filestorage.FileStorage
+
+	Sync() error
+
+	// Different storage knows how to handle comments and tracking
+	Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error)
+}
+
+type baseStorageRuntime struct {
+	meta  RootStorageMeta
+	store filestorage.FileStorage
+}
+
+func (t *baseStorageRuntime) Meta() RootStorageMeta {
+	return t.meta
+}
+
+func (t *baseStorageRuntime) Store() filestorage.FileStorage {
+	return t.store
+}
+
+func (t *baseStorageRuntime) Sync() error {
+	return nil
+}
+
+func (t *baseStorageRuntime) Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error) {
+	return &WriteValueResponse{
+		Code:    500,
+		Message: "unsupportted operation (base)",
+	}, nil
+}
+
+func (t *baseStorageRuntime) setReadOnly(val bool) *baseStorageRuntime {
+	t.meta.ReadOnly = val
+	return t
+}
+
+func (t *baseStorageRuntime) setBuiltin(val bool) *baseStorageRuntime {
+	t.meta.Builtin = val
+	return t
+}
+
+type RootStorageMeta struct {
+	ReadOnly bool          `json:"editable,omitempty"`
+	Builtin  bool          `json:"builtin,omitempty"`
+	Ready    bool          `json:"ready"` // can connect
+	Notice   []data.Notice `json:"notice,omitempty"`
+
+	Config RootStorageConfig `json:"config"`
+}

pkg/services/store/utils.go

@@ -0,0 +1,31 @@
+package store
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/web"
+)
+
+func splitFirstSegment(path string) (string, string) {
+	idx := strings.Index(path, "/")
+	if idx == 0 {
+		path = path[1:]
+		idx = strings.Index(path, "/")
+	}
+
+	if idx > 0 {
+		return path[:idx], path[idx+1:]
+	}
+	return path, ""
+}
+
+func getPathAndScope(c *models.ReqContext) (string, string) {
+	params := web.Params(c.Req)
+	path := params["*"]
+	if path == "" {
+		return "", ""
+	}
+	return splitFirstSegment(filepath.Clean(path))
+}

pkg/services/store/utils_test.go

@@ -0,0 +1,25 @@
+package store
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUtils(t *testing.T) {
+	a, b := splitFirstSegment("")
+	require.Equal(t, "", a)
+	require.Equal(t, "", b)
+
+	a, b = splitFirstSegment("hello")
+	require.Equal(t, "hello", a)
+	require.Equal(t, "", b)
+
+	a, b = splitFirstSegment("hello/world")
+	require.Equal(t, "hello", a)
+	require.Equal(t, "world", b)
+
+	a, b = splitFirstSegment("/hello/world") // strip leading slash
+	require.Equal(t, "hello", a)
+	require.Equal(t, "world", b)
+}

pkg/tsdb/grafanads/grafana.go

@@ -1,20 +1,18 @@
 package grafanads
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
-	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
-	"github.com/grafana/grafana-plugin-sdk-go/experimental"
 	"github.com/grafana/grafana/pkg/components/simplejson"
-	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/searchV2"
+	"github.com/grafana/grafana/pkg/services/store"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb/testdatasource"
 )
@@ -35,27 +33,18 @@ const DatasourceUID = "grafana"
 // This is important to do since otherwise we will only get a
 // not implemented error response from plugin at runtime.
 var (
-	_      backend.QueryDataHandler   = (*Service)(nil)
-	_      backend.CheckHealthHandler = (*Service)(nil)
-	logger                            = log.New("tsdb.grafana")
+	_ backend.QueryDataHandler   = (*Service)(nil)
+	_ backend.CheckHealthHandler = (*Service)(nil)
 )
 
-func ProvideService(cfg *setting.Cfg, search searchV2.SearchService) *Service {
-	return newService(cfg, search)
+func ProvideService(cfg *setting.Cfg, search searchV2.SearchService, store store.StorageService) *Service {
+	return newService(cfg, search, store)
 }
 
-func newService(cfg *setting.Cfg, search searchV2.SearchService) *Service {
+func newService(cfg *setting.Cfg, search searchV2.SearchService, store store.StorageService) *Service {
 	s := &Service{
-		staticRootPath: cfg.StaticRootPath,
-		roots: []string{
-			"testdata",
-			"img/icons",
-			"img/bg",
-			"gazetteer",
-			"maps",
-			"upload", // does not exist yet
-		},
 		search: search,
+		store:  store,
 	}
 
 	return s
@@ -63,10 +52,8 @@ func newService(cfg *setting.Cfg, search searchV2.SearchService) *Service {
 
 // Service exists regardless of user settings
 type Service struct {
-	// path to the public folder
-	staticRootPath string
-	roots          []string
-	search         searchV2.SearchService
+	search searchV2.SearchService
+	store  store.StorageService
 }
 
 func DataSourceModel(orgId int64) *models.DataSource {
@@ -89,9 +76,9 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
 		case queryTypeRandomWalk:
 			response.Responses[q.RefID] = s.doRandomWalk(q)
 		case queryTypeList:
-			response.Responses[q.RefID] = s.doListQuery(q)
+			response.Responses[q.RefID] = s.doListQuery(ctx, q)
 		case queryTypeRead:
-			response.Responses[q.RefID] = s.doReadQuery(q)
+			response.Responses[q.RefID] = s.doReadQuery(ctx, q)
 		case queryTypeSearch:
 			response.Responses[q.RefID] = s.doSearchQuery(ctx, req, q)
 		default:
@@ -111,25 +98,7 @@ func (s *Service) CheckHealth(_ context.Context, _ *backend.CheckHealthRequest)
 	}, nil
 }
 
-func (s *Service) publicPath(path string) (string, error) {
-	if strings.Contains(path, "..") {
-		return "", fmt.Errorf("invalid string")
-	}
-
-	ok := false
-	for _, root := range s.roots {
-		if strings.HasPrefix(path, root) {
-			ok = true
-			break
-		}
-	}
-	if !ok {
-		return "", fmt.Errorf("bad root path")
-	}
-	return filepath.Join(s.staticRootPath, path), nil
-}
-
-func (s *Service) doListQuery(query backend.DataQuery) backend.DataResponse {
+func (s *Service) doListQuery(ctx context.Context, query backend.DataQuery) backend.DataResponse {
 	q := &listQueryModel{}
 	response := backend.DataResponse{}
 	err := json.Unmarshal(query.JSON, &q)
@@ -138,40 +107,17 @@ func (s *Service) doListQuery(query backend.DataQuery) backend.DataResponse {
 		return response
 	}
 
-	if q.Path == "" {
-		count := len(s.roots)
-		names := data.NewFieldFromFieldType(data.FieldTypeString, count)
-		mtype := data.NewFieldFromFieldType(data.FieldTypeString, count)
-		names.Name = "name"
-		mtype.Name = "mediaType"
-		for i, f := range s.roots {
-			names.Set(i, f)
-			mtype.Set(i, "directory")
-		}
-		frame := data.NewFrame("", names, mtype)
-		frame.SetMeta(&data.FrameMeta{
-			Type: data.FrameTypeDirectoryListing,
-		})
-		response.Frames = data.Frames{frame}
-	} else {
-		path, err := s.publicPath(q.Path)
-		if err != nil {
-			response.Error = err
-			return response
-		}
-		frame, err := experimental.GetDirectoryFrame(path, false)
-		if err != nil {
-			response.Error = err
-			return response
-		}
+	path := store.RootPublicStatic + "/" + q.Path
+	frame, err := s.store.List(ctx, nil, path)
+	response.Error = err
+	if frame != nil {
 		response.Frames = data.Frames{frame}
 	}
-
 	return response
 }
 
-func (s *Service) doReadQuery(query backend.DataQuery) backend.DataResponse {
-	q := &listQueryModel{}
+func (s *Service) doReadQuery(ctx context.Context, query backend.DataQuery) backend.DataResponse {
+	q := &readQueryModel{}
 	response := backend.DataResponse{}
 	err := json.Unmarshal(query.JSON, &q)
 	if err != nil {
@@ -184,27 +130,14 @@ func (s *Service) doReadQuery(query backend.DataQuery) backend.DataResponse {
 		return response
 	}
 
-	path, err := s.publicPath(q.Path)
+	path := store.RootPublicStatic + "/" + q.Path
+	file, err := s.store.Read(ctx, nil, path)
 	if err != nil {
 		response.Error = err
 		return response
 	}
 
-	// Can ignore gosec G304 here, because we check the file pattern above
-	// nolint:gosec
-	fileReader, err := os.Open(path)
-	if err != nil {
-		response.Error = fmt.Errorf("failed to read file")
-		return response
-	}
-
-	defer func() {
-		if err := fileReader.Close(); err != nil {
-			logger.Warn("Failed to close file", "err", err, "path", path)
-		}
-	}()
-
-	frame, err := testdatasource.LoadCsvContent(fileReader, filepath.Base(path))
+	frame, err := testdatasource.LoadCsvContent(bytes.NewReader(file.Contents), filepath.Base(path))
 	if err != nil {
 		response.Error = err
 		return response

pkg/tsdb/grafanads/grafana_test.go

@@ -1,43 +0,0 @@
-package grafanads
-
-import (
-	"encoding/json"
-	"path"
-	"testing"
-
-	"github.com/grafana/grafana/pkg/services/searchV2"
-	"github.com/grafana/grafana/pkg/setting"
-
-	"github.com/grafana/grafana-plugin-sdk-go/backend"
-	"github.com/grafana/grafana-plugin-sdk-go/experimental"
-	"github.com/stretchr/testify/require"
-)
-
-func asJSON(v interface{}) json.RawMessage {
-	b, _ := json.Marshal(v)
-	return b
-}
-
-func TestReadFolderListing(t *testing.T) {
-	ds := newService(&setting.Cfg{StaticRootPath: "../../../public"}, searchV2.NewStubSearchService())
-	dr := ds.doListQuery(backend.DataQuery{
-		QueryType: "x",
-		JSON: asJSON(listQueryModel{
-			Path: "testdata",
-		}),
-	})
-	err := experimental.CheckGoldenDataResponse(path.Join("testdata", "list.golden.txt"), &dr, true)
-	require.NoError(t, err)
-}
-
-func TestReadCSVFile(t *testing.T) {
-	ds := newService(&setting.Cfg{StaticRootPath: "../../../public"}, searchV2.NewStubSearchService())
-	dr := ds.doReadQuery(backend.DataQuery{
-		QueryType: "x",
-		JSON: asJSON(readQueryModel{
-			Path: "testdata/js_libraries.csv",
-		}),
-	})
-	err := experimental.CheckGoldenDataResponse(path.Join("testdata", "jslib.golden.txt"), &dr, true)
-	require.NoError(t, err)
-}

pkg/tsdb/grafanads/testdata/list.golden.txt

@@ -1,25 +0,0 @@
-🌟 This was machine generated.  Do not edit. 🌟
-
-Frame[0] {
-    "type": "directory-listing",
-    "pathSeparator": "/"
-}
-Name: 
-Dimensions: 2 Fields by 7 Rows
-+--------------------------+------------------+
-| Name: name               | Name: media-type |
-| Labels:                  | Labels:          |
-| Type: []string           | Type: []string   |
-+--------------------------+------------------+
-| browser_marketshare.csv  |                  |
-| flight_info_by_state.csv |                  |
-| gdp_per_capita.csv       |                  |
-| js_libraries.csv         |                  |
-| ohlc_dogecoin.csv        |                  |
-| population_by_state.csv  |                  |
-| weight_height.csv        |                  |
-+--------------------------+------------------+
-
-
-====== TEST DATA RESPONSE (arrow base64) ======
-FRAME=QVJST1cxAAD/////uAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAKQAAAADAAAATAAAACgAAAAEAAAA0P7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADw/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAABD///8IAAAAPAAAADAAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInBhdGhTZXBhcmF0b3IiOiIvIn0AAAAABAAAAG1ldGEAAAAAAgAAAHwAAAAEAAAAnv///xQAAABAAAAAQAAAAAAAAAU8AAAAAQAAAAQAAACM////CAAAABQAAAAKAAAAbWVkaWEtdHlwZQAABAAAAG5hbWUAAAAAAAAAAIj///8KAAAAbWVkaWEtdHlwZQAAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAG5hbWUAAAAABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAABAAAAG5hbWUAAAAA/////9gAAAAUAAAAAAAAAAwAFgAUABMADAAEAAwAAADQAAAAAAAAABQAAAAAAAADAwAKABgADAAIAAQACgAAABQAAAB4AAAABwAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAACQAAAAAAAAALAAAAAAAAAAAAAAAAAAAACwAAAAAAAAACAAAAAAAAAA0AAAAAAAAAAAAAAAAAAAAAAAAAACAAAABwAAAAAAAAAAAAAAAAAAAAcAAAAAAAAAAAAAAAAAAAAAAAAAFwAAAC8AAABBAAAAUQAAAGIAAAB5AAAAigAAAGJyb3dzZXJfbWFya2V0c2hhcmUuY3N2ZmxpZ2h0X2luZm9fYnlfc3RhdGUuY3N2Z2RwX3Blcl9jYXBpdGEuY3N2anNfbGlicmFyaWVzLmNzdm9obGNfZG9nZWNvaW4uY3N2cG9wdWxhdGlvbl9ieV9zdGF0ZS5jc3Z3ZWlnaHRfaGVpZ2h0LmNzdgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAMAAQAAAMgBAAAAAAAA4AAAAAAAAADQAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAApAAAAAMAAABMAAAAKAAAAAQAAADQ/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAPD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAEP///wgAAAA8AAAAMAAAAHsidHlwZSI6ImRpcmVjdG9yeS1saXN0aW5nIiwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAfAAAAAQAAACe////FAAAAEAAAABAAAAAAAAABTwAAAABAAAABAAAAIz///8IAAAAFAAAAAoAAABtZWRpYS10eXBlAAAEAAAAbmFtZQAAAAAAAAAAiP///woAAABtZWRpYS10eXBlAAAAABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEgAAAAAAAAFRAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAbmFtZQAAAAAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAEAAAAbmFtZQAAAADoAQAAQVJST1cx