Storage: add basic storage service (#46604)

pkg/services/featuremgmt/registry.go

@@ -162,6 +162,17 @@ var (
 			Description: "Lock database during migrations",
 			State:       FeatureStateBeta,
 		},
+		{
+			Name:        "storage",
+			Description: "Configurable storage for dashboards, datasources, and resources",
+			State:       FeatureStateAlpha,
+		},
+		{
+			Name:            "storageLocalUpload",
+			Description:     "allow uploads to local storage",
+			State:           FeatureStateAlpha,
+			RequiresDevMode: true,
+		},
 		{
 			Name:            "azureMonitorResourcePickerForMetrics",
 			Description:     "New UI for Azure Monitor Metrics Query",

pkg/services/featuremgmt/toggles_gen.go

@@ -123,6 +123,14 @@ const (
 	// Lock database during migrations
 	FlagMigrationLocking = "migrationLocking"
 
+	// FlagStorage
+	// Configurable storage for dashboards, datasources, and resources
+	FlagStorage = "storage"
+
+	// FlagStorageLocalUpload
+	// allow uploads to local storage
+	FlagStorageLocalUpload = "storageLocalUpload"
+
 	// FlagAzureMonitorResourcePickerForMetrics
 	// New UI for Azure Monitor Metrics Query
 	FlagAzureMonitorResourcePickerForMetrics = "azureMonitorResourcePickerForMetrics"

pkg/services/store/config.go

@@ -0,0 +1,53 @@
+package store
+
+type RootStorageConfig struct {
+	Type   string `json:"type"`
+	Prefix string `json:"prefix"`
+	Name   string `json:"name"`
+
+	// Depending on type, these will be configured
+	Disk *StorageLocalDiskConfig `json:"disk,omitempty"`
+	Git  *StorageGitConfig       `json:"git,omitempty"`
+	SQL  *StorageSQLConfig       `json:"sql,omitempty"`
+	S3   *StorageS3Config        `json:"s3,omitempty"`
+	GCS  *StorageGCSConfig       `json:"gcs,omitempty"`
+}
+
+type StorageLocalDiskConfig struct {
+	Path  string   `json:"path"`
+	Roots []string `json:"roots,omitempty"` // null is everything
+}
+
+type StorageGitConfig struct {
+	Remote string `json:"remote"`
+	Branch string `json:"branch"`
+	Root   string `json:"root"` // subfolder within the remote
+
+	// Pull interval?
+	// Requires pull request?
+	RequirePullRequest bool `json:"requirePullRequest"`
+
+	// SECURE JSON :grimicing:
+	AccessToken string `json:"accessToken,omitempty"` // Simplest auth method for github
+}
+
+type StorageSQLConfig struct {
+	// no custom settings
+}
+
+type StorageS3Config struct {
+	Bucket string `json:"bucket"`
+	Folder string `json:"folder"`
+
+	// SECURE!!!
+	AccessKey string `json:"accessKey"`
+	SecretKey string `json:"secretKey"`
+	Region    string `json:"region"`
+}
+
+type StorageGCSConfig struct {
+	Bucket string `json:"bucket"`
+	Folder string `json:"folder"`
+
+	CredentialsFile string `json:"credentialsFile"`
+}

pkg/services/store/http.go

@@ -0,0 +1,71 @@
+package store
+
+import (
+	"github.com/grafana/grafana/pkg/api/response"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/web"
+)
+
+// HTTPStorageService passes raw HTTP requests to a well typed storage service
+type HTTPStorageService interface {
+	List(c *models.ReqContext) response.Response
+	Read(c *models.ReqContext) response.Response
+	Delete(c *models.ReqContext) response.Response
+	Upload(c *models.ReqContext) response.Response
+}
+
+type httpStorage struct {
+	store StorageService
+}
+
+func ProvideHTTPService(store StorageService) HTTPStorageService {
+	return &httpStorage{
+		store: store,
+	}
+}
+
+func (s *httpStorage) Upload(c *models.ReqContext) response.Response {
+	action := "Upload"
+	scope, path := getPathAndScope(c)
+
+	return response.JSON(200, map[string]string{
+		"action": action,
+		"scope":  scope,
+		"path":   path,
+	})
+}
+
+func (s *httpStorage) Read(c *models.ReqContext) response.Response {
+	action := "Read"
+	scope, path := getPathAndScope(c)
+
+	return response.JSON(200, map[string]string{
+		"action": action,
+		"scope":  scope,
+		"path":   path,
+	})
+}
+
+func (s *httpStorage) Delete(c *models.ReqContext) response.Response {
+	action := "Delete"
+	scope, path := getPathAndScope(c)
+
+	return response.JSON(200, map[string]string{
+		"action": action,
+		"scope":  scope,
+		"path":   path,
+	})
+}
+
+func (s *httpStorage) List(c *models.ReqContext) response.Response {
+	params := web.Params(c.Req)
+	path := params["*"]
+	frame, err := s.store.List(c.Req.Context(), c.SignedInUser, path)
+	if err != nil {
+		return response.Error(400, "error reading path", err)
+	}
+	if frame == nil {
+		return response.Error(404, "not found", nil)
+	}
+	return response.JSONStreaming(200, frame)
+}

pkg/services/store/service.go

@@ -0,0 +1,88 @@
+package store
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/infra/filestorage"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/registry"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+var grafanaStorageLogger = log.New("grafanaStorageLogger")
+
+const RootPublicStatic = "public-static"
+
+type StorageService interface {
+	registry.BackgroundService
+
+	// List folder contents
+	List(ctx context.Context, user *models.SignedInUser, path string) (*data.Frame, error)
+
+	// Read raw file contents out of the store
+	Read(ctx context.Context, user *models.SignedInUser, path string) (*filestorage.File, error)
+}
+
+type standardStorageService struct {
+	sql  *sqlstore.SQLStore
+	tree *nestedTree
+}
+
+func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, cfg *setting.Cfg) StorageService {
+	roots := []storageRuntime{
+		newDiskStorage(RootPublicStatic, "Public static files", &StorageLocalDiskConfig{
+			Path: cfg.StaticRootPath,
+			Roots: []string{
+				"/testdata/",
+				// "/img/icons/",
+				// "/img/bg/",
+				"/img/",
+				"/gazetteer/",
+				"/maps/",
+			},
+		}).setReadOnly(true).setBuiltin(true),
+	}
+
+	storage := filepath.Join(cfg.DataPath, "storage")
+	_ = os.MkdirAll(storage, 0700)
+
+	if features.IsEnabled(featuremgmt.FlagStorageLocalUpload) {
+		roots = append(roots, newDiskStorage("upload", "Local file upload", &StorageLocalDiskConfig{
+			Path: filepath.Join(storage, "upload"),
+		}))
+	}
+	s := newStandardStorageService(roots)
+	s.sql = sql
+	return s
+}
+
+func newStandardStorageService(roots []storageRuntime) *standardStorageService {
+	res := &nestedTree{
+		roots: roots,
+	}
+	res.init()
+	return &standardStorageService{
+		tree: res,
+	}
+}
+
+func (s *standardStorageService) Run(ctx context.Context) error {
+	grafanaStorageLogger.Info("storage starting")
+	return nil
+}
+
+func (s *standardStorageService) List(ctx context.Context, user *models.SignedInUser, path string) (*data.Frame, error) {
+	// apply access control here
+	return s.tree.ListFolder(ctx, path)
+}
+
+func (s *standardStorageService) Read(ctx context.Context, user *models.SignedInUser, path string) (*filestorage.File, error) {
+	// TODO: permission check!
+	return s.tree.GetFile(ctx, path)
+}

pkg/services/store/service_test.go

@@ -0,0 +1,47 @@
+package store
+
+import (
+	"bytes"
+	"context"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/experimental"
+	"github.com/grafana/grafana/pkg/tsdb/testdatasource"
+	"github.com/stretchr/testify/require"
+)
+
+func TestListFiles(t *testing.T) {
+	publicRoot, err := filepath.Abs("../../../public")
+	require.NoError(t, err)
+	roots := []storageRuntime{
+		newDiskStorage("public", "Public static files", &StorageLocalDiskConfig{
+			Path: publicRoot,
+			Roots: []string{
+				"/testdata/",
+				"/img/icons/",
+				"/img/bg/",
+				"/gazetteer/",
+				"/maps/",
+				"/upload/",
+			},
+		}).setReadOnly(true).setBuiltin(true),
+	}
+
+	store := newStandardStorageService(roots)
+	frame, err := store.List(context.Background(), nil, "public/testdata")
+	require.NoError(t, err)
+
+	err = experimental.CheckGoldenFrame(path.Join("testdata", "public_testdata.golden.txt"), frame, true)
+	require.NoError(t, err)
+
+	file, err := store.Read(context.Background(), nil, "public/testdata/js_libraries.csv")
+	require.NoError(t, err)
+	require.NotNil(t, file)
+
+	frame, err = testdatasource.LoadCsvContent(bytes.NewReader(file.Contents), file.Name)
+	require.NoError(t, err)
+	err = experimental.CheckGoldenFrame(path.Join("testdata", "public_testdata_js_libraries.golden.txt"), frame, true)
+	require.NoError(t, err)
+}

pkg/services/store/storage_disk.go

@@ -0,0 +1,91 @@
+package store
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/infra/filestorage"
+	"gocloud.dev/blob"
+)
+
+const rootStorageTypeDisk = "disk"
+
+type rootStorageDisk struct {
+	baseStorageRuntime
+
+	settings *StorageLocalDiskConfig
+}
+
+func newDiskStorage(prefix string, name string, cfg *StorageLocalDiskConfig) *rootStorageDisk {
+	if cfg == nil {
+		cfg = &StorageLocalDiskConfig{}
+	}
+
+	meta := RootStorageMeta{
+		Config: RootStorageConfig{
+			Type:   rootStorageTypeDisk,
+			Prefix: prefix,
+			Name:   name,
+			Disk:   cfg,
+		},
+	}
+	if prefix == "" {
+		meta.Notice = append(meta.Notice, data.Notice{
+			Severity: data.NoticeSeverityError,
+			Text:     "Missing prefix",
+		})
+	}
+	if cfg.Path == "" {
+		meta.Notice = append(meta.Notice, data.Notice{
+			Severity: data.NoticeSeverityError,
+			Text:     "Missing path configuration",
+		})
+	}
+	s := &rootStorageDisk{}
+
+	if meta.Notice == nil {
+		path := fmt.Sprintf("file://%s", cfg.Path)
+		bucket, err := blob.OpenBucket(context.Background(), path)
+		if err != nil {
+			grafanaStorageLogger.Warn("error loading storage", "prefix", prefix, "err", err)
+			meta.Notice = append(meta.Notice, data.Notice{
+				Severity: data.NoticeSeverityError,
+				Text:     "Failed to initialize storage",
+			})
+		} else {
+			s.store = filestorage.NewCdkBlobStorage(grafanaStorageLogger,
+				bucket, "",
+				filestorage.NewPathFilters(cfg.Roots, nil, nil, nil))
+
+			meta.Ready = true // exists!
+		}
+	}
+
+	s.meta = meta
+	s.settings = cfg
+	return s
+}
+
+func (s *rootStorageDisk) Sync() error {
+	return nil // already in sync
+}
+
+// with local disk user metadata and messages are lost
+func (s *rootStorageDisk) Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error) {
+	byteAray := []byte(cmd.Body)
+
+	path := cmd.Path
+	if !strings.HasPrefix(path, filestorage.Delimiter) {
+		path = filestorage.Delimiter + path
+	}
+	err := s.store.Upsert(ctx, &filestorage.UpsertFileCommand{
+		Path:     path,
+		Contents: &byteAray,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &WriteValueResponse{Code: 200}, nil
+}

pkg/services/store/testdata/public_testdata.golden.txt

@@ -0,0 +1,27 @@
+🌟 This was machine generated.  Do not edit. 🌟
+
+Frame[0] {
+    "type": "directory-listing",
+    "custom": {
+        "HasMore": false
+    }
+}
+Name: 
+Dimensions: 3 Fields by 7 Rows
++--------------------------+-------------------------+---------------+
+| Name: name               | Name: mediaType         | Name: size    |
+| Labels:                  | Labels:                 | Labels:       |
+| Type: []string           | Type: []string          | Type: []int64 |
++--------------------------+-------------------------+---------------+
+| browser_marketshare.csv  | text/csv; charset=utf-8 | 355           |
+| flight_info_by_state.csv | text/csv; charset=utf-8 | 681           |
+| gdp_per_capita.csv       | text/csv; charset=utf-8 | 4116          |
+| js_libraries.csv         | text/csv; charset=utf-8 | 179           |
+| ohlc_dogecoin.csv        | text/csv; charset=utf-8 | 191804        |
+| population_by_state.csv  | text/csv; charset=utf-8 | 138           |
+| weight_height.csv        | text/csv; charset=utf-8 | 418121        |
++--------------------------+-------------------------+---------------+
+
+
+====== TEST DATA RESPONSE (arrow base64) ======
+FRAME=QVJST1cxAAD/////WAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAKgAAAADAAAATAAAACgAAAAEAAAAMP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABQ/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAHD+//8IAAAAQAAAADcAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsImN1c3RvbSI6eyJIYXNNb3JlIjpmYWxzZX19AAQAAABtZXRhAAAAAAMAAAAYAQAApAAAAAQAAAAG////FAAAAHAAAAB4AAAAAAAAAnwAAAACAAAALAAAAAQAAAD4/v//CAAAABAAAAAEAAAAc2l6ZQAAAAAEAAAAbmFtZQAAAAAc////CAAAABwAAAAQAAAAeyJ1bml0IjoiYnl0ZXMifQAAAAAGAAAAY29uZmlnAAAAAAAACAAMAAgABwAIAAAAAAAAAUAAAAAEAAAAc2l6ZQAAAACi////FAAAAEAAAABAAAAAAAAABTwAAAABAAAABAAAAJD///8IAAAAFAAAAAkAAABtZWRpYVR5cGUAAAAEAAAAbmFtZQAAAAAAAAAAjP///wkAAABtZWRpYVR5cGUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABuYW1lAAAAAAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP////8IAQAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAsAEAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAmAAAAAcAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAigAAAAAAAACwAAAAAAAAAAAAAAAAAAAAsAAAAAAAAAAgAAAAAAAAANAAAAAAAAAAoQAAAAAAAAB4AQAAAAAAAAAAAAAAAAAAeAEAAAAAAAA4AAAAAAAAAAAAAAADAAAABwAAAAAAAAAAAAAAAAAAAAcAAAAAAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAABcAAAAvAAAAQQAAAFEAAABiAAAAeQAAAIoAAABicm93c2VyX21hcmtldHNoYXJlLmNzdmZsaWdodF9pbmZvX2J5X3N0YXRlLmNzdmdkcF9wZXJfY2FwaXRhLmNzdmpzX2xpYnJhcmllcy5jc3ZvaGxjX2RvZ2Vjb2luLmNzdnBvcHVsYXRpb25fYnlfc3RhdGUuY3N2d2VpZ2h0X2hlaWdodC5jc3YAAAAAAAAAAAAAFwAAAC4AAABFAAAAXAAAAHMAAACKAAAAoQAAAHRleHQvY3N2OyBjaGFyc2V0PXV0Zi04dGV4dC9jc3Y7IGNoYXJzZXQ9dXRmLTh0ZXh0L2NzdjsgY2hhcnNldD11dGYtOHRleHQvY3N2OyBjaGFyc2V0PXV0Zi04dGV4dC9jc3Y7IGNoYXJzZXQ9dXRmLTh0ZXh0L2NzdjsgY2hhcnNldD11dGYtOHRleHQvY3N2OyBjaGFyc2V0PXV0Zi04AAAAAAAAAGMBAAAAAAAAqQIAAAAAAAAUEAAAAAAAALMAAAAAAAAAPO0CAAAAAACKAAAAAAAAAElhBgAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAQAAQAAAGgCAAAAAAAAEAEAAAAAAACwAQAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAqAAAAAMAAABMAAAAKAAAAAQAAAAw/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAFD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAcP7//wgAAABAAAAANwAAAHsidHlwZSI6ImRpcmVjdG9yeS1saXN0aW5nIiwiY3VzdG9tIjp7Ikhhc01vcmUiOmZhbHNlfX0ABAAAAG1ldGEAAAAAAwAAABgBAACkAAAABAAAAAb///8UAAAAcAAAAHgAAAAAAAACfAAAAAIAAAAsAAAABAAAAPj+//8IAAAAEAAAAAQAAABzaXplAAAAAAQAAABuYW1lAAAAABz///8IAAAAHAAAABAAAAB7InVuaXQiOiJieXRlcyJ9AAAAAAYAAABjb25maWcAAAAAAAAIAAwACAAHAAgAAAAAAAABQAAAAAQAAABzaXplAAAAAKL///8UAAAAQAAAAEAAAAAAAAAFPAAAAAEAAAAEAAAAkP///wgAAAAUAAAACQAAAG1lZGlhVHlwZQAAAAQAAABuYW1lAAAAAAAAAACM////CQAAAG1lZGlhVHlwZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAG5hbWUAAAAABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAABAAAAG5hbWUAAAAAiAIAAEFSUk9XMQ==

pkg/services/store/testdata/public_testdata_js_libraries.golden.txt

@@ -0,0 +1,21 @@
+🌟 This was machine generated.  Do not edit. 🌟
+
+Frame[0] 
+Name: js_libraries.csv
+Dimensions: 4 Fields by 6 Rows
++-----------------+--------------------+----------------+----------------+
+| Name: Library   | Name: Github Stars | Name: Forks    | Name: Watchers |
+| Labels:         | Labels:            | Labels:        | Labels:        |
+| Type: []*string | Type: []*int64     | Type: []*int64 | Type: []*int64 |
++-----------------+--------------------+----------------+----------------+
+| React.js        | 169000             | 34000          | 6700           |
+| Vue             | 184000             | 29100          | 6300           |
+| Angular         | 73400              | 19300          | 3200           |
+| JQuery          | 54900              | 20000          | 3300           |
+| Meteor          | 42400              | 5200           | 1700           |
+| Aurelia         | 11600              | 684            | 442            |
++-----------------+--------------------+----------------+----------------+
+
+
+====== TEST DATA RESPONSE (arrow base64) ======
+FRAME=QVJST1cxAAD/////WAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAGAAAAACAAAAKAAAAAQAAAAs/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAEz+//8IAAAAHAAAABAAAABqc19saWJyYXJpZXMuY3N2AAAAAAQAAABuYW1lAAAAAAQAAABgAQAA1AAAAHAAAAAEAAAAwv7//xQAAABAAAAAQAAAAAAAAgFEAAAAAQAAAAQAAACw/v//CAAAABQAAAAIAAAAV2F0Y2hlcnMAAAAABAAAAG5hbWUAAAAAAAAAADT///8AAAABQAAAAAgAAABXYXRjaGVycwAAAAAq////FAAAADwAAAA8AAAAAAACAUAAAAABAAAABAAAABj///8IAAAAEAAAAAUAAABGb3JrcwAAAAQAAABuYW1lAAAAAAAAAACY////AAAAAUAAAAAFAAAARm9ya3MAAACK////FAAAAEQAAABMAAAAAAACAVAAAAABAAAABAAAAHj///8IAAAAGAAAAAwAAABHaXRodWIgU3RhcnMAAAAABAAAAG5hbWUAAAAAAAAAAAgADAAIAAcACAAAAAAAAAFAAAAADAAAAEdpdGh1YiBTdGFycwAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAABQFEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAcAAABMaWJyYXJ5AAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAcAAABMaWJyYXJ5AP////8oAQAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAA2AAAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAqAAAAAYAAAAAAAAAAAAAAAkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAcAAAAAAAAACAAAAAAAAAAJQAAAAAAAABIAAAAAAAAAAAAAAAAAAAASAAAAAAAAAAwAAAAAAAAAHgAAAAAAAAAAAAAAAAAAAB4AAAAAAAAADAAAAAAAAAAqAAAAAAAAAAAAAAAAAAAAKgAAAAAAAAAMAAAAAAAAAAAAAAABAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAsAAAASAAAAGAAAAB4AAAAlAAAAAAAAAFJlYWN0LmpzVnVlQW5ndWxhckpRdWVyeU1ldGVvckF1cmVsaWEAAAAolAIAAAAAAMDOAgAAAAAAuB4BAAAAAAB01gAAAAAAAKClAAAAAAAAUC0AAAAAAADQhAAAAAAAAKxxAAAAAAAAZEsAAAAAAAAgTgAAAAAAAFAUAAAAAAAArAIAAAAAAAAsGgAAAAAAAJwYAAAAAAAAgAwAAAAAAADkDAAAAAAAAKQGAAAAAAAAugEAAAAAAAAQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAABAABAAAAaAIAAAAAAAAwAQAAAAAAANgAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABgAAAAAgAAACgAAAAEAAAALP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABM/v//CAAAABwAAAAQAAAAanNfbGlicmFyaWVzLmNzdgAAAAAEAAAAbmFtZQAAAAAEAAAAYAEAANQAAABwAAAABAAAAML+//8UAAAAQAAAAEAAAAAAAAIBRAAAAAEAAAAEAAAAsP7//wgAAAAUAAAACAAAAFdhdGNoZXJzAAAAAAQAAABuYW1lAAAAAAAAAAA0////AAAAAUAAAAAIAAAAV2F0Y2hlcnMAAAAAKv///xQAAAA8AAAAPAAAAAAAAgFAAAAAAQAAAAQAAAAY////CAAAABAAAAAFAAAARm9ya3MAAAAEAAAAbmFtZQAAAAAAAAAAmP///wAAAAFAAAAABQAAAEZvcmtzAAAAiv///xQAAABEAAAATAAAAAAAAgFQAAAAAQAAAAQAAAB4////CAAAABgAAAAMAAAAR2l0aHViIFN0YXJzAAAAAAQAAABuYW1lAAAAAAAAAAAIAAwACAAHAAgAAAAAAAABQAAAAAwAAABHaXRodWIgU3RhcnMAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAARAAAAEgAAAAAAAUBRAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAHAAAATGlicmFyeQAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAHAAAATGlicmFyeQCIAgAAQVJST1cx

pkg/services/store/tree.go

@@ -0,0 +1,106 @@
+package store
+
+import (
+	"context"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/infra/filestorage"
+)
+
+type nestedTree struct {
+	roots  []storageRuntime
+	lookup map[string]filestorage.FileStorage
+}
+
+var (
+	_ storageTree = (*nestedTree)(nil)
+)
+
+func (t *nestedTree) init() {
+	t.lookup = make(map[string]filestorage.FileStorage, len(t.roots))
+	for _, root := range t.roots {
+		t.lookup[root.Meta().Config.Prefix] = root.Store()
+	}
+}
+
+func (t *nestedTree) getRoot(path string) (filestorage.FileStorage, string) {
+	if path == "" {
+		return nil, ""
+	}
+
+	rootKey, path := splitFirstSegment(path)
+	root, ok := t.lookup[rootKey]
+	if !ok || root == nil {
+		return nil, path // not found or not ready
+	}
+	return root, filestorage.Delimiter + path
+}
+
+func (t *nestedTree) GetFile(ctx context.Context, path string) (*filestorage.File, error) {
+	if path == "" {
+		return nil, nil // not found
+	}
+
+	root, path := t.getRoot(path)
+	if root == nil {
+		return nil, nil // not found (or not ready)
+	}
+	return root.Get(ctx, path)
+}
+
+func (t *nestedTree) ListFolder(ctx context.Context, path string) (*data.Frame, error) {
+	if path == "" || path == "/" {
+		count := len(t.roots)
+		title := data.NewFieldFromFieldType(data.FieldTypeString, count)
+		names := data.NewFieldFromFieldType(data.FieldTypeString, count)
+		mtype := data.NewFieldFromFieldType(data.FieldTypeString, count)
+		title.Name = "title"
+		names.Name = "name"
+		mtype.Name = "mediaType"
+		for i, f := range t.roots {
+			names.Set(i, f.Meta().Config.Prefix)
+			title.Set(i, f.Meta().Config.Name)
+			mtype.Set(i, "directory")
+		}
+		frame := data.NewFrame("", names, title, mtype)
+		frame.SetMeta(&data.FrameMeta{
+			Type: data.FrameTypeDirectoryListing,
+		})
+		return frame, nil
+	}
+
+	root, path := t.getRoot(path)
+	if root == nil {
+		return nil, nil // not found (or not ready)
+	}
+
+	listResponse, err := root.List(ctx, path, nil, &filestorage.ListOptions{Recursive: false, WithFolders: true, WithFiles: true})
+
+	if err != nil {
+		return nil, err
+	}
+
+	count := len(listResponse.Files)
+	names := data.NewFieldFromFieldType(data.FieldTypeString, count)
+	mtype := data.NewFieldFromFieldType(data.FieldTypeString, count)
+	fsize := data.NewFieldFromFieldType(data.FieldTypeInt64, count)
+	names.Name = "name"
+	mtype.Name = "mediaType"
+	fsize.Name = "size"
+	fsize.Config = &data.FieldConfig{
+		Unit: "bytes",
+	}
+	for i, f := range listResponse.Files {
+		names.Set(i, f.Name)
+		mtype.Set(i, f.MimeType)
+		fsize.Set(i, f.Size)
+	}
+	frame := data.NewFrame("", names, mtype, fsize)
+	frame.SetMeta(&data.FrameMeta{
+		Type: data.FrameTypeDirectoryListing,
+		Custom: map[string]interface{}{
+			"HasMore": listResponse.HasMore,
+		},
+	})
+	return frame, nil
+}

pkg/services/store/types.go

@@ -0,0 +1,92 @@
+package store
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/infra/filestorage"
+	"github.com/grafana/grafana/pkg/models"
+)
+
+type WriteValueRequest struct {
+	Path    string
+	User    *models.SignedInUser
+	Body    json.RawMessage `json:"body,omitempty"`
+	Message string          `json:"message,omitempty"`
+	Title   string          `json:"title,omitempty"`  // For PRs
+	Action  string          `json:"action,omitempty"` // pr | save
+}
+
+type WriteValueResponse struct {
+	Code    int    `json:"code,omitempty"`
+	Message string `json:"message,omitempty"`
+	URL     string `json:"url,omitempty"`
+	Hash    string `json:"hash,omitempty"`
+	Branch  string `json:"branch,omitempty"`
+	Pending bool   `json:"pending,omitempty"`
+	Size    int64  `json:"size,omitempty"`
+}
+
+type storageTree interface {
+	GetFile(ctx context.Context, path string) (*filestorage.File, error)
+	ListFolder(ctx context.Context, path string) (*data.Frame, error)
+}
+
+//-------------------------------------------
+// INTERNAL
+//-------------------------------------------
+
+type storageRuntime interface {
+	Meta() RootStorageMeta
+
+	Store() filestorage.FileStorage
+
+	Sync() error
+
+	// Different storage knows how to handle comments and tracking
+	Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error)
+}
+
+type baseStorageRuntime struct {
+	meta  RootStorageMeta
+	store filestorage.FileStorage
+}
+
+func (t *baseStorageRuntime) Meta() RootStorageMeta {
+	return t.meta
+}
+
+func (t *baseStorageRuntime) Store() filestorage.FileStorage {
+	return t.store
+}
+
+func (t *baseStorageRuntime) Sync() error {
+	return nil
+}
+
+func (t *baseStorageRuntime) Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error) {
+	return &WriteValueResponse{
+		Code:    500,
+		Message: "unsupportted operation (base)",
+	}, nil
+}
+
+func (t *baseStorageRuntime) setReadOnly(val bool) *baseStorageRuntime {
+	t.meta.ReadOnly = val
+	return t
+}
+
+func (t *baseStorageRuntime) setBuiltin(val bool) *baseStorageRuntime {
+	t.meta.Builtin = val
+	return t
+}
+
+type RootStorageMeta struct {
+	ReadOnly bool          `json:"editable,omitempty"`
+	Builtin  bool          `json:"builtin,omitempty"`
+	Ready    bool          `json:"ready"` // can connect
+	Notice   []data.Notice `json:"notice,omitempty"`
+
+	Config RootStorageConfig `json:"config"`
+}

pkg/services/store/utils.go

@@ -0,0 +1,31 @@
+package store
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/web"
+)
+
+func splitFirstSegment(path string) (string, string) {
+	idx := strings.Index(path, "/")
+	if idx == 0 {
+		path = path[1:]
+		idx = strings.Index(path, "/")
+	}
+
+	if idx > 0 {
+		return path[:idx], path[idx+1:]
+	}
+	return path, ""
+}
+
+func getPathAndScope(c *models.ReqContext) (string, string) {
+	params := web.Params(c.Req)
+	path := params["*"]
+	if path == "" {
+		return "", ""
+	}
+	return splitFirstSegment(filepath.Clean(path))
+}

pkg/services/store/utils_test.go

@@ -0,0 +1,25 @@
+package store
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUtils(t *testing.T) {
+	a, b := splitFirstSegment("")
+	require.Equal(t, "", a)
+	require.Equal(t, "", b)
+
+	a, b = splitFirstSegment("hello")
+	require.Equal(t, "hello", a)
+	require.Equal(t, "", b)
+
+	a, b = splitFirstSegment("hello/world")
+	require.Equal(t, "hello", a)
+	require.Equal(t, "world", b)
+
+	a, b = splitFirstSegment("/hello/world") // strip leading slash
+	require.Equal(t, "hello", a)
+	require.Equal(t, "world", b)
+}