From 68df83c86dc38ce7f51637786c92d4051487f64b Mon Sep 17 00:00:00 2001 From: Will Browne Date: Tue, 7 Mar 2023 15:47:02 +0000 Subject: [PATCH] Plugins: Add Plugin FS abstraction (#63734) * unexport pluginDir from dto * first pass * tidy * naming + add mutex * add dupe checking * fix func typo * interface + move logic from renderer * remote finder * remote signing * fix tests * tidy up * tidy markdown logic * split changes * fix tests * slim interface down * fix status code * tidy exec path func * fixup * undo changes * remove unused func * remove unused func * fix goimports * fetch remotely * simultaneous support * fix linter * use var * add exception for gosec warning * fixup * fix tests * tidy * rework cfg pattern * simplify * PR feedback * fix dupe field * remove g304 nolint * apply PR feedback * remove unnecessary gosec nolint * fix finder loop and update comment * fix map alloc * fix test * remove commented code --- pkg/api/plugins.go | 9 +- pkg/api/plugins_test.go | 106 ++-- pkg/plugins/ifaces.go | 18 + pkg/plugins/localfiles.go | 91 ++++ pkg/plugins/manager/fakes/fakes.go | 25 + pkg/plugins/manager/installer_test.go | 10 +- pkg/plugins/manager/loader/finder/finder.go | 96 +--- .../manager/loader/finder/finder_test.go | 121 ----- pkg/plugins/manager/loader/finder/fs.go | 286 +++++++++++ pkg/plugins/manager/loader/finder/fs_test.go | 485 ++++++++++++++++++ pkg/plugins/manager/loader/finder/ifaces.go | 11 + .../loader/initializer/initializer_test.go | 13 +- pkg/plugins/manager/loader/loader.go | 224 ++------ pkg/plugins/manager/loader/loader_test.go | 404 ++++++--------- .../manager/manager_integration_test.go | 19 +- pkg/plugins/manager/signature/manifest.go | 149 +++--- .../manager/signature/manifest_test.go | 51 +- pkg/plugins/manager/sources/sources.go | 2 +- pkg/plugins/manager/store/store.go | 5 +- pkg/plugins/manager/store/store_test.go | 12 +- pkg/plugins/plugins.go | 72 ++- pkg/services/updatechecker/plugins_test.go | 5 +- 22 files changed, 1344 insertions(+), 870 deletions(-) create mode 100644 pkg/plugins/localfiles.go delete mode 100644 pkg/plugins/manager/loader/finder/finder_test.go create mode 100644 pkg/plugins/manager/loader/finder/fs.go create mode 100644 pkg/plugins/manager/loader/finder/fs_test.go create mode 100644 pkg/plugins/manager/loader/finder/ifaces.go diff --git a/pkg/api/plugins.go b/pkg/api/plugins.go index ebc4301971c..5bb0e0834c2 100644 --- a/pkg/api/plugins.go +++ b/pkg/api/plugins.go @@ -271,17 +271,20 @@ func (hs *HTTPServer) GetPluginMarkdown(c *contextmodel.ReqContext) response.Res if err != nil { var notFound plugins.NotFoundError if errors.As(err, ¬Found) { - return response.Error(404, notFound.Error(), nil) + return response.Error(http.StatusNotFound, notFound.Error(), nil) } - return response.Error(500, "Could not get markdown file", err) + return response.Error(http.StatusInternalServerError, "Could not get markdown file", err) } // fallback try readme if len(content) == 0 { content, err = hs.pluginMarkdown(c.Req.Context(), pluginID, "readme") if err != nil { - return response.Error(501, "Could not get markdown file", err) + if errors.Is(err, plugins.ErrFileNotExist) { + return response.Error(http.StatusNotFound, plugins.ErrFileNotExist.Error(), nil) + } + return response.Error(http.StatusNotImplemented, "Could not get markdown file", err) } } diff --git a/pkg/api/plugins_test.go b/pkg/api/plugins_test.go index db1d1075ac9..bcaddeba8c5 100644 --- a/pkg/api/plugins_test.go +++ b/pkg/api/plugins_test.go @@ -20,7 +20,6 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/infra/log/logtest" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins/config" "github.com/grafana/grafana/pkg/plugins/pluginscdn" @@ -270,14 +269,9 @@ func Test_GetPluginAssets(t *testing.T) { requestedFile := filepath.Clean(tmpFile.Name()) t.Run("Given a request for an existing plugin file", func(t *testing.T) { - p := &plugins.Plugin{ - JSONData: plugins.JSONData{ - ID: pluginID, - }, - PluginDir: pluginDir, - } + p := createPluginDTO(plugins.JSONData{ID: pluginID}, plugins.External, plugins.NewLocalFS(map[string]struct{}{requestedFile: {}}, filepath.Dir(requestedFile))) service := &plugins.FakePluginStore{ - PluginList: []plugins.PluginDTO{p.ToDTO()}, + PluginList: []plugins.PluginDTO{p}, } url := fmt.Sprintf("/public/plugins/%s/%s", pluginID, requestedFile) @@ -291,7 +285,7 @@ func Test_GetPluginAssets(t *testing.T) { }) t.Run("Given a request for a relative path", func(t *testing.T) { - p := createPluginDTO(plugins.JSONData{ID: pluginID}, plugins.External, pluginDir) + p := createPluginDTO(plugins.JSONData{ID: pluginID}, plugins.External, plugins.NewLocalFS(map[string]struct{}{}, "")) service := &plugins.FakePluginStore{ PluginList: []plugins.PluginDTO{p}, } @@ -305,8 +299,26 @@ func Test_GetPluginAssets(t *testing.T) { }) }) + t.Run("Given a request for an existing plugin file that is not listed as a signature covered file", func(t *testing.T) { + p := createPluginDTO(plugins.JSONData{ID: pluginID}, plugins.Core, plugins.NewLocalFS(map[string]struct{}{ + requestedFile: {}, + }, "")) + service := &plugins.FakePluginStore{ + PluginList: []plugins.PluginDTO{p}, + } + + url := fmt.Sprintf("/public/plugins/%s/%s", pluginID, requestedFile) + pluginAssetScenario(t, "When calling GET on", url, "/public/plugins/:pluginId/*", + setting.NewCfg(), service, func(sc *scenarioContext) { + callGetPluginAsset(sc) + + require.Equal(t, 200, sc.resp.Code) + assert.Equal(t, expectedBody, sc.resp.Body.String()) + }) + }) + t.Run("Given a request for an non-existing plugin file", func(t *testing.T) { - p := createPluginDTO(plugins.JSONData{ID: pluginID}, plugins.External, pluginDir) + p := createPluginDTO(plugins.JSONData{ID: pluginID}, plugins.External, plugins.NewLocalFS(map[string]struct{}{}, "")) service := &plugins.FakePluginStore{ PluginList: []plugins.PluginDTO{p}, } @@ -329,7 +341,6 @@ func Test_GetPluginAssets(t *testing.T) { service := &plugins.FakePluginStore{ PluginList: []plugins.PluginDTO{}, } - l := &logtest.Fake{} requestedFile := "nonExistent" url := fmt.Sprintf("/public/plugins/%s/%s", pluginID, requestedFile) @@ -342,29 +353,6 @@ func Test_GetPluginAssets(t *testing.T) { require.NoError(t, err) require.Equal(t, 404, sc.resp.Code) require.Equal(t, "Plugin not found", respJson["message"]) - require.Zero(t, l.WarnLogs.Calls) - }) - }) - - t.Run("Given a request for a core plugin's file", func(t *testing.T) { - service := &plugins.FakePluginStore{ - PluginList: []plugins.PluginDTO{ - { - JSONData: plugins.JSONData{ID: pluginID}, - Class: plugins.Core, - }, - }, - } - l := &logtest.Fake{} - - url := fmt.Sprintf("/public/plugins/%s/%s", pluginID, requestedFile) - pluginAssetScenario(t, "When calling GET on", url, "/public/plugins/:pluginId/*", - setting.NewCfg(), service, func(sc *scenarioContext) { - callGetPluginAsset(sc) - - require.Equal(t, 200, sc.resp.Code) - require.Equal(t, expectedBody, sc.resp.Body.String()) - require.Zero(t, l.WarnLogs.Calls) }) }) } @@ -546,40 +534,19 @@ func (c *fakePluginClient) QueryData(ctx context.Context, req *backend.QueryData } func Test_PluginsList_AccessControl(t *testing.T) { - p1 := &plugins.Plugin{ - PluginDir: "/grafana/plugins/test-app/dist", - Class: plugins.External, - DefaultNavURL: "/plugins/test-app/page/test", - Signature: plugins.SignatureUnsigned, - Module: "plugins/test-app/module", - BaseURL: "public/plugins/test-app", - JSONData: plugins.JSONData{ - ID: "test-app", - Type: plugins.App, - Name: "test-app", - Info: plugins.Info{ - Version: "1.0.0", - }, - }, - } - p2 := &plugins.Plugin{ - PluginDir: "/grafana/public/app/plugins/datasource/mysql", - Class: plugins.Core, - Pinned: false, - Signature: plugins.SignatureInternal, - Module: "app/plugins/datasource/mysql/module", - BaseURL: "public/app/plugins/datasource/mysql", - JSONData: plugins.JSONData{ - ID: "mysql", - Type: plugins.DataSource, - Name: "MySQL", + p1 := createPluginDTO(plugins.JSONData{ + ID: "test-app", Type: "app", Name: "test-app", + Info: plugins.Info{ + Version: "1.0.0", + }}, plugins.External, plugins.NewLocalFS(map[string]struct{}{}, "")) + p2 := createPluginDTO( + plugins.JSONData{ID: "mysql", Type: "datasource", Name: "MySQL", Info: plugins.Info{ Author: plugins.InfoLink{Name: "Grafana Labs", URL: "https://grafana.com"}, Description: "Data source for MySQL databases", - }, - }, - } - pluginStore := plugins.FakePluginStore{PluginList: []plugins.PluginDTO{p1.ToDTO(), p2.ToDTO()}} + }}, plugins.Core, plugins.NewLocalFS(map[string]struct{}{}, "")) + + pluginStore := plugins.FakePluginStore{PluginList: []plugins.PluginDTO{p1, p2}} pluginSettings := pluginsettings.FakePluginSettings{Plugins: map[string]*pluginsettings.DTO{ "test-app": {ID: 0, OrgID: 1, PluginID: "test-app", PluginVersion: "1.0.0", Enabled: true}, @@ -630,11 +597,12 @@ func Test_PluginsList_AccessControl(t *testing.T) { } } -func createPluginDTO(jd plugins.JSONData, class plugins.Class, pluginDir string) plugins.PluginDTO { +func createPluginDTO(jd plugins.JSONData, class plugins.Class, files plugins.FS) plugins.PluginDTO { p := &plugins.Plugin{ - JSONData: jd, - Class: class, - PluginDir: pluginDir, + JSONData: jd, + Class: class, + FS: files, } + return p.ToDTO() } diff --git a/pkg/plugins/ifaces.go b/pkg/plugins/ifaces.go index 9c8d36f03ab..fac68172d8a 100644 --- a/pkg/plugins/ifaces.go +++ b/pkg/plugins/ifaces.go @@ -2,6 +2,7 @@ package plugins import ( "context" + "io/fs" "github.com/grafana/grafana-plugin-sdk-go/backend" @@ -38,6 +39,23 @@ type UpdateInfo struct { PluginZipURL string } +type FS interface { + fs.FS + + Base() string + Files() []string +} + +type FoundBundle struct { + Primary FoundPlugin + Children []*FoundPlugin +} + +type FoundPlugin struct { + JSONData JSONData + FS FS +} + // Client is used to communicate with backend plugin implementations. type Client interface { backend.QueryDataHandler diff --git a/pkg/plugins/localfiles.go b/pkg/plugins/localfiles.go new file mode 100644 index 00000000000..805e62bcb4d --- /dev/null +++ b/pkg/plugins/localfiles.go @@ -0,0 +1,91 @@ +package plugins + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/grafana/grafana/pkg/util" +) + +var _ fs.FS = (*LocalFS)(nil) + +type LocalFS struct { + m map[string]*LocalFile + basePath string +} + +func NewLocalFS(m map[string]struct{}, basePath string) LocalFS { + pfs := make(map[string]*LocalFile, len(m)) + for k := range m { + pfs[k] = &LocalFile{ + path: k, + } + } + + return LocalFS{ + m: pfs, + basePath: basePath, + } +} + +func (f LocalFS) Open(name string) (fs.File, error) { + cleanPath, err := util.CleanRelativePath(name) + if err != nil { + return nil, err + } + + if kv, exists := f.m[filepath.Join(f.basePath, cleanPath)]; exists { + if kv.f != nil { + return kv.f, nil + } + return os.Open(kv.path) + } + return nil, ErrFileNotExist +} + +func (f LocalFS) Base() string { + return f.basePath +} + +func (f LocalFS) Files() []string { + var files []string + for p := range f.m { + r, err := filepath.Rel(f.basePath, p) + if strings.Contains(r, "..") || err != nil { + continue + } + files = append(files, r) + } + + return files +} + +var _ fs.File = (*LocalFile)(nil) + +type LocalFile struct { + f *os.File + path string +} + +func (p *LocalFile) Stat() (fs.FileInfo, error) { + return os.Stat(p.path) +} + +func (p *LocalFile) Read(bytes []byte) (int, error) { + var err error + p.f, err = os.Open(p.path) + if err != nil { + return 0, err + } + return p.f.Read(bytes) +} + +func (p *LocalFile) Close() error { + if p.f != nil { + return p.f.Close() + } + p.f = nil + return nil +} diff --git a/pkg/plugins/manager/fakes/fakes.go b/pkg/plugins/manager/fakes/fakes.go index 6ac997b0138..394cce815eb 100644 --- a/pkg/plugins/manager/fakes/fakes.go +++ b/pkg/plugins/manager/fakes/fakes.go @@ -4,6 +4,7 @@ import ( "archive/zip" "context" "fmt" + "io/fs" "sync" "github.com/grafana/grafana-plugin-sdk-go/backend" @@ -351,6 +352,30 @@ func (f *FakeRoleRegistry) DeclarePluginRoles(_ context.Context, _ string, _ str return f.ExpectedErr } +type FakePluginFiles struct { + FS fs.FS + + base string +} + +func NewFakePluginFiles(base string) *FakePluginFiles { + return &FakePluginFiles{ + base: base, + } +} + +func (f *FakePluginFiles) Open(name string) (fs.File, error) { + return f.FS.Open(name) +} + +func (f *FakePluginFiles) Base() string { + return f.base +} + +func (f *FakePluginFiles) Files() []string { + return []string{} +} + type FakeSources struct { ListFunc func(_ context.Context) []plugins.PluginSource } diff --git a/pkg/plugins/manager/installer_test.go b/pkg/plugins/manager/installer_test.go index f7084ab7d35..0b768eac7f8 100644 --- a/pkg/plugins/manager/installer_test.go +++ b/pkg/plugins/manager/installer_test.go @@ -22,13 +22,11 @@ func TestPluginManager_Add_Remove(t *testing.T) { const ( pluginID, v1 = "test-panel", "1.0.0" zipNameV1 = "test-panel-1.0.0.zip" - pluginDirV1 = "/data/plugin/test-panel-1.0.0" ) // mock a plugin to be returned automatically by the plugin loader pluginV1 := createPlugin(t, pluginID, plugins.External, true, true, func(plugin *plugins.Plugin) { plugin.Info.Version = v1 - plugin.PluginDir = pluginDirV1 }) mockZipV1 := &zip.ReadCloser{Reader: zip.Reader{File: []*zip.File{{ FileHeader: zip.FileHeader{Name: zipNameV1}, @@ -63,7 +61,6 @@ func TestPluginManager_Add_Remove(t *testing.T) { }, RegisterFunc: func(_ context.Context, pluginID, pluginDir string) error { require.Equal(t, pluginV1.ID, pluginID) - require.Equal(t, pluginV1.PluginDir, pluginDir) return nil }, Store: map[string]struct{}{}, @@ -88,14 +85,12 @@ func TestPluginManager_Add_Remove(t *testing.T) { t.Run("Update plugin to different version", func(t *testing.T) { const ( - v2 = "2.0.0" - zipNameV2 = "test-panel-2.0.0.zip" - pluginDirV2 = "/data/plugin/test-panel-2.0.0" + v2 = "2.0.0" + zipNameV2 = "test-panel-2.0.0.zip" ) // mock a plugin to be returned automatically by the plugin loader pluginV2 := createPlugin(t, pluginID, plugins.External, true, true, func(plugin *plugins.Plugin) { plugin.Info.Version = v2 - plugin.PluginDir = pluginDirV2 }) mockZipV2 := &zip.ReadCloser{Reader: zip.Reader{File: []*zip.File{{ @@ -126,7 +121,6 @@ func TestPluginManager_Add_Remove(t *testing.T) { } fs.RegisterFunc = func(_ context.Context, pluginID, pluginDir string) error { require.Equal(t, pluginV2.ID, pluginID) - require.Equal(t, pluginV2.PluginDir, pluginDir) return nil } diff --git a/pkg/plugins/manager/loader/finder/finder.go b/pkg/plugins/manager/loader/finder/finder.go index 8813978dab2..6697e0437c8 100644 --- a/pkg/plugins/manager/loader/finder/finder.go +++ b/pkg/plugins/manager/loader/finder/finder.go @@ -1,90 +1,44 @@ package finder import ( - "errors" - "fmt" - "os" - "path/filepath" + "context" - "github.com/grafana/grafana/pkg/infra/fs" + "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins/log" - "github.com/grafana/grafana/pkg/util" ) -var walk = util.Walk - -type Finder struct { - log log.Logger +type Service struct { + local *FS + log log.Logger } -func New() Finder { - return Finder{log: log.New("plugin.finder")} +func NewService() *Service { + logger := log.New("plugin.finder") + return &Service{ + local: newFS(logger), + log: logger, + } } -func (f *Finder) Find(pluginPaths []string) ([]string, error) { - var pluginJSONPaths []string +func (f *Service) Find(ctx context.Context, pluginPaths ...string) ([]*plugins.FoundBundle, error) { + if len(pluginPaths) == 0 { + return []*plugins.FoundBundle{}, nil + } + fbs := make(map[string][]*plugins.FoundBundle) for _, path := range pluginPaths { - exists, err := fs.Exists(path) + local, err := f.local.Find(ctx, path) if err != nil { - f.log.Warn("Error occurred when checking if plugin directory exists", "path", path, "err", err) - } - if !exists { - f.log.Warn("Skipping finding plugins as directory does not exist", "path", path) + f.log.Warn("Error occurred when trying to find plugin", "path", path) continue } - - paths, err := f.getAbsPluginJSONPaths(path) - if err != nil { - return nil, err - } - pluginJSONPaths = append(pluginJSONPaths, paths...) + fbs[path] = local } - return pluginJSONPaths, nil -} - -func (f *Finder) getAbsPluginJSONPaths(path string) ([]string, error) { - var pluginJSONPaths []string - - var err error - path, err = filepath.Abs(path) - if err != nil { - return []string{}, err - } - - if err := walk(path, true, true, - func(currentPath string, fi os.FileInfo, err error) error { - if err != nil { - if errors.Is(err, os.ErrNotExist) { - f.log.Error("Couldn't scan directory since it doesn't exist", "pluginDir", path, "err", err) - return nil - } - if errors.Is(err, os.ErrPermission) { - f.log.Error("Couldn't scan directory due to lack of permissions", "pluginDir", path, "err", err) - return nil - } - - return fmt.Errorf("filepath.Walk reported an error for %q: %w", currentPath, err) - } - - if fi.Name() == "node_modules" { - return util.ErrWalkSkipDir - } - - if fi.IsDir() { - return nil - } - - if fi.Name() != "plugin.json" { - return nil - } - - pluginJSONPaths = append(pluginJSONPaths, currentPath) - return nil - }); err != nil { - return []string{}, err - } - - return pluginJSONPaths, nil + var found []*plugins.FoundBundle + for _, fb := range fbs { + found = append(found, fb...) + } + + return found, nil } diff --git a/pkg/plugins/manager/loader/finder/finder_test.go b/pkg/plugins/manager/loader/finder/finder_test.go deleted file mode 100644 index 045a42186c0..00000000000 --- a/pkg/plugins/manager/loader/finder/finder_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package finder - -import ( - "errors" - "fmt" - "os" - "strings" - "testing" - - "github.com/grafana/grafana/pkg/plugins/log" - "github.com/grafana/grafana/pkg/util" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestFinder_Find(t *testing.T) { - testCases := []struct { - name string - pluginDirs []string - expectedPathSuffix []string - err error - }{ - { - name: "Dir with single plugin", - pluginDirs: []string{"../../testdata/valid-v2-signature"}, - expectedPathSuffix: []string{"/pkg/plugins/manager/testdata/valid-v2-signature/plugin/plugin.json"}, - }, - { - name: "Dir with nested plugins", - pluginDirs: []string{"../../testdata/duplicate-plugins"}, - expectedPathSuffix: []string{ - "/pkg/plugins/manager/testdata/duplicate-plugins/nested/nested/plugin.json", - "/pkg/plugins/manager/testdata/duplicate-plugins/nested/plugin.json", - }, - }, - { - name: "Dir with single plugin which has symbolic link root directory", - pluginDirs: []string{"../../testdata/symbolic-plugin-dirs"}, - expectedPathSuffix: []string{"/pkg/plugins/manager/testdata/includes-symlinks/plugin.json"}, - }, - { - name: "Multiple plugin dirs", - pluginDirs: []string{"../../testdata/duplicate-plugins", "../../testdata/invalid-v1-signature"}, - expectedPathSuffix: []string{ - "/pkg/plugins/manager/testdata/duplicate-plugins/nested/nested/plugin.json", - "/pkg/plugins/manager/testdata/duplicate-plugins/nested/plugin.json", - "/pkg/plugins/manager/testdata/invalid-v1-signature/plugin/plugin.json"}, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - f := New() - pluginPaths, err := f.Find(tc.pluginDirs) - if (err != nil) && !errors.Is(err, tc.err) { - t.Errorf("Find() error = %v, expected error %v", err, tc.err) - return - } - - assert.Equal(t, len(tc.expectedPathSuffix), len(pluginPaths)) - for i := 0; i < len(tc.expectedPathSuffix); i++ { - assert.True(t, strings.HasSuffix(pluginPaths[i], tc.expectedPathSuffix[i])) - } - }) - } -} - -func TestFinder_getAbsPluginJSONPaths(t *testing.T) { - t.Run("When scanning a folder that doesn't exists shouldn't return an error", func(t *testing.T) { - origWalk := walk - walk = func(path string, followSymlinks, detectSymlinkInfiniteLoop bool, walkFn util.WalkFunc) error { - return walkFn(path, nil, os.ErrNotExist) - } - t.Cleanup(func() { - walk = origWalk - }) - - finder := &Finder{ - log: log.NewTestLogger(), - } - - paths, err := finder.getAbsPluginJSONPaths("test") - require.NoError(t, err) - require.Empty(t, paths) - }) - - t.Run("When scanning a folder that lacks permission shouldn't return an error", func(t *testing.T) { - origWalk := walk - walk = func(path string, followSymlinks, detectSymlinkInfiniteLoop bool, walkFn util.WalkFunc) error { - return walkFn(path, nil, os.ErrPermission) - } - t.Cleanup(func() { - walk = origWalk - }) - - finder := &Finder{ - log: log.NewTestLogger(), - } - - paths, err := finder.getAbsPluginJSONPaths("test") - require.NoError(t, err) - require.Empty(t, paths) - }) - - t.Run("When scanning a folder that returns a non-handled error should return that error", func(t *testing.T) { - origWalk := walk - walk = func(path string, followSymlinks, detectSymlinkInfiniteLoop bool, walkFn util.WalkFunc) error { - return walkFn(path, nil, fmt.Errorf("random error")) - } - t.Cleanup(func() { - walk = origWalk - }) - - finder := &Finder{ - log: log.NewTestLogger(), - } - - paths, err := finder.getAbsPluginJSONPaths("test") - require.Error(t, err) - require.Empty(t, paths) - }) -} diff --git a/pkg/plugins/manager/loader/finder/fs.go b/pkg/plugins/manager/loader/finder/fs.go new file mode 100644 index 00000000000..629ddbbcbe9 --- /dev/null +++ b/pkg/plugins/manager/loader/finder/fs.go @@ -0,0 +1,286 @@ +package finder + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/grafana/grafana/pkg/infra/fs" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/plugins/log" + "github.com/grafana/grafana/pkg/services/org" + "github.com/grafana/grafana/pkg/util" +) + +var walk = util.Walk + +var ( + ErrInvalidPluginJSON = errors.New("did not find valid type or id properties in plugin.json") + ErrInvalidPluginJSONFilePath = errors.New("invalid plugin.json filepath was provided") +) + +type FS struct { + log log.Logger +} + +func newFS(logger log.Logger) *FS { + return &FS{log: logger.New("fs")} +} + +func (f *FS) Find(_ context.Context, pluginPaths ...string) ([]*plugins.FoundBundle, error) { + if len(pluginPaths) == 0 { + return []*plugins.FoundBundle{}, nil + } + + var pluginJSONPaths []string + for _, path := range pluginPaths { + exists, err := fs.Exists(path) + if err != nil { + f.log.Warn("Skipping finding plugins as an error occurred", "path", path, "err", err) + continue + } + if !exists { + f.log.Warn("Skipping finding plugins as directory does not exist", "path", path) + continue + } + + paths, err := f.getAbsPluginJSONPaths(path) + if err != nil { + return nil, err + } + pluginJSONPaths = append(pluginJSONPaths, paths...) + } + + // load plugin.json files and map directory to JSON data + foundPlugins := make(map[string]plugins.JSONData) + for _, pluginJSONPath := range pluginJSONPaths { + plugin, err := f.readPluginJSON(pluginJSONPath) + if err != nil { + f.log.Warn("Skipping plugin loading as its plugin.json could not be read", "path", pluginJSONPath, "err", err) + continue + } + + pluginJSONAbsPath, err := filepath.Abs(pluginJSONPath) + if err != nil { + f.log.Warn("Skipping plugin loading as absolute plugin.json path could not be calculated", "pluginID", plugin.ID, "err", err) + continue + } + + if _, dupe := foundPlugins[filepath.Dir(pluginJSONAbsPath)]; dupe { + f.log.Warn("Skipping plugin loading as it's a duplicate", "pluginID", plugin.ID) + continue + } + foundPlugins[filepath.Dir(pluginJSONAbsPath)] = plugin + } + + var res = make(map[string]*plugins.FoundBundle) + for pluginDir, data := range foundPlugins { + files, err := collectFilesWithin(pluginDir) + if err != nil { + return nil, err + } + + res[pluginDir] = &plugins.FoundBundle{ + Primary: plugins.FoundPlugin{ + JSONData: data, + FS: plugins.NewLocalFS(files, pluginDir), + }, + } + } + + var result []*plugins.FoundBundle + for dir := range foundPlugins { + ancestors := strings.Split(dir, string(filepath.Separator)) + ancestors = ancestors[0 : len(ancestors)-1] + + pluginPath := "" + if runtime.GOOS != "windows" && filepath.IsAbs(dir) { + pluginPath = "/" + } + add := true + for _, ancestor := range ancestors { + pluginPath = filepath.Join(pluginPath, ancestor) + if _, ok := foundPlugins[pluginPath]; ok { + if fp, exists := res[pluginPath]; exists { + fp.Children = append(fp.Children, &res[dir].Primary) + add = false + break + } + } + } + if add { + result = append(result, res[dir]) + } + } + + return result, nil +} + +func (f *FS) getAbsPluginJSONPaths(path string) ([]string, error) { + var pluginJSONPaths []string + + var err error + path, err = filepath.Abs(path) + if err != nil { + return []string{}, err + } + + if err = walk(path, true, true, + func(currentPath string, fi os.FileInfo, err error) error { + if err != nil { + if errors.Is(err, os.ErrNotExist) { + f.log.Error("Couldn't scan directory since it doesn't exist", "pluginDir", path, "err", err) + return nil + } + if errors.Is(err, os.ErrPermission) { + f.log.Error("Couldn't scan directory due to lack of permissions", "pluginDir", path, "err", err) + return nil + } + + return fmt.Errorf("filepath.Walk reported an error for %q: %w", currentPath, err) + } + + if fi.Name() == "node_modules" { + return util.ErrWalkSkipDir + } + + if fi.IsDir() { + return nil + } + + if fi.Name() != "plugin.json" { + return nil + } + + pluginJSONPaths = append(pluginJSONPaths, currentPath) + return nil + }); err != nil { + return []string{}, err + } + + return pluginJSONPaths, nil +} + +func (f *FS) readPluginJSON(pluginJSONPath string) (plugins.JSONData, error) { + f.log.Debug("Loading plugin", "path", pluginJSONPath) + + if !strings.EqualFold(filepath.Ext(pluginJSONPath), ".json") { + return plugins.JSONData{}, ErrInvalidPluginJSONFilePath + } + + absPluginJSONPath, err := filepath.Abs(pluginJSONPath) + if err != nil { + return plugins.JSONData{}, err + } + + // Wrapping in filepath.Clean to properly handle + // gosec G304 Potential file inclusion via variable rule. + reader, err := os.Open(filepath.Clean(absPluginJSONPath)) + if err != nil { + return plugins.JSONData{}, err + } + defer func() { + if reader == nil { + return + } + if err = reader.Close(); err != nil { + f.log.Warn("Failed to close JSON file", "path", pluginJSONPath, "err", err) + } + }() + + plugin := plugins.JSONData{} + if err = json.NewDecoder(reader).Decode(&plugin); err != nil { + return plugins.JSONData{}, err + } + + if err = validatePluginJSON(plugin); err != nil { + return plugins.JSONData{}, err + } + + if plugin.ID == "grafana-piechart-panel" { + plugin.Name = "Pie Chart (old)" + } + + if len(plugin.Dependencies.Plugins) == 0 { + plugin.Dependencies.Plugins = []plugins.Dependency{} + } + + if plugin.Dependencies.GrafanaVersion == "" { + plugin.Dependencies.GrafanaVersion = "*" + } + + for _, include := range plugin.Includes { + if include.Role == "" { + include.Role = org.RoleViewer + } + } + + return plugin, nil +} + +func validatePluginJSON(data plugins.JSONData) error { + if data.ID == "" || !data.Type.IsValid() { + return ErrInvalidPluginJSON + } + return nil +} + +func collectFilesWithin(dir string) (map[string]struct{}, error) { + files := map[string]struct{}{} + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.Mode()&os.ModeSymlink == os.ModeSymlink { + symlinkPath, err := filepath.EvalSymlinks(path) + if err != nil { + return err + } + + symlink, err := os.Stat(symlinkPath) + if err != nil { + return err + } + + // verify that symlinked file is within plugin directory + p, err := filepath.Rel(dir, symlinkPath) + if err != nil { + return err + } + if p == ".." || strings.HasPrefix(p, ".."+string(filepath.Separator)) { + return fmt.Errorf("file '%s' not inside of plugin directory", p) + } + + // skip adding symlinked directories + if symlink.IsDir() { + return nil + } + } + + // skip directories + if info.IsDir() { + return nil + } + + // verify that file is within plugin directory + file, err := filepath.Rel(dir, path) + if err != nil { + return err + } + if strings.HasPrefix(file, ".."+string(filepath.Separator)) { + return fmt.Errorf("file '%s' not inside of plugin directory", file) + } + + files[path] = struct{}{} + + return nil + }) + + return files, err +} diff --git a/pkg/plugins/manager/loader/finder/fs_test.go b/pkg/plugins/manager/loader/finder/fs_test.go new file mode 100644 index 00000000000..1954d41c146 --- /dev/null +++ b/pkg/plugins/manager/loader/finder/fs_test.go @@ -0,0 +1,485 @@ +package finder + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/plugins/log" + "github.com/grafana/grafana/pkg/services/org" + "github.com/grafana/grafana/pkg/util" +) + +func TestFinder_Find(t *testing.T) { + testData, err := filepath.Abs("../../testdata") + if err != nil { + require.NoError(t, err) + } + testCases := []struct { + name string + pluginDirs []string + expectedBundles []*plugins.FoundBundle + err error + }{ + { + name: "Dir with single plugin", + pluginDirs: []string{filepath.Join(testData, "valid-v2-signature")}, + expectedBundles: []*plugins.FoundBundle{ + { + Primary: plugins.FoundPlugin{ + JSONData: plugins.JSONData{ + ID: "test-datasource", + Type: plugins.DataSource, + Name: "Test", + Info: plugins.Info{ + Author: plugins.InfoLink{ + Name: "Will Browne", + URL: "https://willbrowne.com", + }, + Description: "Test", + Version: "1.0.0", + }, + Dependencies: plugins.Dependencies{ + GrafanaVersion: "*", + Plugins: []plugins.Dependency{}, + }, + State: plugins.AlphaRelease, + Backend: true, + Executable: "test", + }, + FS: plugins.NewLocalFS(map[string]struct{}{ + filepath.Join(testData, "valid-v2-signature/plugin/plugin.json"): {}, + filepath.Join(testData, "valid-v2-signature/plugin/MANIFEST.txt"): {}, + }, filepath.Join(testData, "valid-v2-signature/plugin")), + }, + }, + }, + }, + { + name: "Dir with nested plugins", + pluginDirs: []string{"../../testdata/duplicate-plugins"}, + expectedBundles: []*plugins.FoundBundle{ + { + Primary: plugins.FoundPlugin{ + JSONData: plugins.JSONData{ + ID: "test-app", + Type: plugins.DataSource, + Name: "Parent", + Info: plugins.Info{ + Author: plugins.InfoLink{ + Name: "Grafana Labs", + URL: "http://grafana.com", + }, + Description: "Parent plugin", + Version: "1.0.0", + Updated: "2020-10-20", + }, + Dependencies: plugins.Dependencies{ + GrafanaVersion: "*", + Plugins: []plugins.Dependency{}, + }, + }, + FS: plugins.NewLocalFS(map[string]struct{}{ + filepath.Join(testData, "duplicate-plugins/nested/plugin.json"): {}, + filepath.Join(testData, "duplicate-plugins/nested/MANIFEST.txt"): {}, + filepath.Join(testData, "duplicate-plugins/nested/nested/plugin.json"): {}, + filepath.Join(testData, "duplicate-plugins/nested/nested/MANIFEST.txt"): {}, + }, filepath.Join(testData, "duplicate-plugins/nested")), + }, + Children: []*plugins.FoundPlugin{ + { + JSONData: plugins.JSONData{ + ID: "test-app", + Type: plugins.DataSource, + Name: "Child", + Info: plugins.Info{ + Author: plugins.InfoLink{ + Name: "Grafana Labs", + URL: "http://grafana.com", + }, + Description: "Child plugin", + Version: "1.0.0", + Updated: "2020-10-20", + }, + Dependencies: plugins.Dependencies{ + GrafanaVersion: "*", + Plugins: []plugins.Dependency{}, + }, + }, + FS: plugins.NewLocalFS(map[string]struct{}{ + filepath.Join(testData, "duplicate-plugins/nested/nested/plugin.json"): {}, + filepath.Join(testData, "duplicate-plugins/nested/nested/MANIFEST.txt"): {}, + }, filepath.Join(testData, "duplicate-plugins/nested/nested")), + }, + }, + }, + }, + }, + { + name: "Dir with single plugin which has symbolic link root directory", + pluginDirs: []string{"../../testdata/symbolic-plugin-dirs"}, + expectedBundles: []*plugins.FoundBundle{ + { + Primary: plugins.FoundPlugin{ + JSONData: plugins.JSONData{ + ID: "test-app", + Type: plugins.App, + Name: "Test App", + Info: plugins.Info{ + Author: plugins.InfoLink{ + Name: "Test Inc.", + URL: "http://test.com", + }, + Description: "Official Grafana Test App & Dashboard bundle", + Version: "1.0.0", + Links: []plugins.InfoLink{ + {Name: "Project site", URL: "http://project.com"}, + {Name: "License & Terms", URL: "http://license.com"}, + }, + Updated: "2015-02-10", + Logos: plugins.Logos{ + Small: "img/logo_small.png", + Large: "img/logo_large.png", + }, + Screenshots: []plugins.Screenshots{ + {Name: "img1", Path: "img/screenshot1.png"}, + {Name: "img2", Path: "img/screenshot2.png"}, + }, + }, + Dependencies: plugins.Dependencies{ + GrafanaVersion: "3.x.x", + Plugins: []plugins.Dependency{ + {ID: "graphite", Type: "datasource", Name: "Graphite", Version: "1.0.0"}, + {ID: "graph", Type: "panel", Name: "Graph", Version: "1.0.0"}, + }, + }, + Includes: []*plugins.Includes{ + { + Name: "Nginx Connections", + Path: "dashboards/connections.json", + Type: "dashboard", + Role: "Viewer", + }, + { + Name: "Nginx Memory", + Path: "dashboards/memory.json", + Type: "dashboard", + Role: "Viewer", + }, + {Name: "Nginx Panel", Type: "panel", Role: "Viewer"}, + {Name: "Nginx Datasource", Type: "datasource", Role: "Viewer"}, + }, + }, + FS: plugins.NewLocalFS(map[string]struct{}{ + filepath.Join(testData, "includes-symlinks/MANIFEST.txt"): {}, + filepath.Join(testData, "includes-symlinks/dashboards/connections.json"): {}, + filepath.Join(testData, "includes-symlinks/dashboards/extra/memory.json"): {}, + filepath.Join(testData, "includes-symlinks/plugin.json"): {}, + filepath.Join(testData, "includes-symlinks/symlink_to_txt"): {}, + filepath.Join(testData, "includes-symlinks/text.txt"): {}, + }, filepath.Join(testData, "includes-symlinks")), + }, + }, + }, + }, + { + name: "Multiple plugin dirs", + pluginDirs: []string{"../../testdata/duplicate-plugins", "../../testdata/invalid-v1-signature"}, + expectedBundles: []*plugins.FoundBundle{{ + Primary: plugins.FoundPlugin{ + JSONData: plugins.JSONData{ + ID: "test-app", + Type: plugins.DataSource, + Name: "Parent", + Info: plugins.Info{ + Author: plugins.InfoLink{ + Name: "Grafana Labs", + URL: "http://grafana.com", + }, + Description: "Parent plugin", + Version: "1.0.0", + Updated: "2020-10-20", + }, + Dependencies: plugins.Dependencies{ + GrafanaVersion: "*", + Plugins: []plugins.Dependency{}, + }, + }, + FS: plugins.NewLocalFS(map[string]struct{}{ + filepath.Join(testData, "duplicate-plugins/nested/plugin.json"): {}, + filepath.Join(testData, "duplicate-plugins/nested/MANIFEST.txt"): {}, + filepath.Join(testData, "duplicate-plugins/nested/nested/plugin.json"): {}, + filepath.Join(testData, "duplicate-plugins/nested/nested/MANIFEST.txt"): {}, + }, filepath.Join(testData, "duplicate-plugins/nested")), + }, + Children: []*plugins.FoundPlugin{ + { + JSONData: plugins.JSONData{ + ID: "test-app", + Type: plugins.DataSource, + Name: "Child", + Info: plugins.Info{ + Author: plugins.InfoLink{ + Name: "Grafana Labs", + URL: "http://grafana.com", + }, + Description: "Child plugin", + Version: "1.0.0", + Updated: "2020-10-20", + }, + Dependencies: plugins.Dependencies{ + GrafanaVersion: "*", + Plugins: []plugins.Dependency{}, + }, + }, + FS: plugins.NewLocalFS(map[string]struct{}{ + filepath.Join(testData, "duplicate-plugins/nested/nested/plugin.json"): {}, + filepath.Join(testData, "duplicate-plugins/nested/nested/MANIFEST.txt"): {}, + }, filepath.Join(testData, "duplicate-plugins/nested/nested")), + }, + }, + }, + { + Primary: plugins.FoundPlugin{ + JSONData: plugins.JSONData{ + ID: "test-datasource", + Type: plugins.DataSource, + Name: "Test", + Info: plugins.Info{ + Author: plugins.InfoLink{ + Name: "Grafana Labs", + URL: "https://grafana.com", + }, + Description: "Test", + }, + Dependencies: plugins.Dependencies{ + GrafanaVersion: "*", + Plugins: []plugins.Dependency{}, + }, + State: plugins.AlphaRelease, + Backend: true, + }, + FS: plugins.NewLocalFS(map[string]struct{}{ + filepath.Join(testData, "invalid-v1-signature/plugin/plugin.json"): {}, + filepath.Join(testData, "invalid-v1-signature/plugin/MANIFEST.txt"): {}, + }, filepath.Join(testData, "invalid-v1-signature/plugin")), + }, + }, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + f := newFS(log.NewTestLogger()) + pluginBundles, err := f.Find(context.Background(), tc.pluginDirs...) + if (err != nil) && !errors.Is(err, tc.err) { + t.Errorf("Find() error = %v, expected error %v", err, tc.err) + return + } + + // to ensure we can compare with expected + sort.SliceStable(pluginBundles, func(i, j int) bool { + return pluginBundles[i].Primary.JSONData.ID < pluginBundles[j].Primary.JSONData.ID + }) + + if !cmp.Equal(pluginBundles, tc.expectedBundles, localFSComparer) { + t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(pluginBundles, tc.expectedBundles, localFSComparer)) + } + }) + } +} + +func TestFinder_getAbsPluginJSONPaths(t *testing.T) { + t.Run("When scanning a folder that doesn't exists shouldn't return an error", func(t *testing.T) { + origWalk := walk + walk = func(path string, followSymlinks, detectSymlinkInfiniteLoop bool, walkFn util.WalkFunc) error { + return walkFn(path, nil, os.ErrNotExist) + } + t.Cleanup(func() { + walk = origWalk + }) + + finder := newFS(log.NewTestLogger()) + paths, err := finder.getAbsPluginJSONPaths("test") + require.NoError(t, err) + require.Empty(t, paths) + }) + + t.Run("When scanning a folder that lacks permission shouldn't return an error", func(t *testing.T) { + origWalk := walk + walk = func(path string, followSymlinks, detectSymlinkInfiniteLoop bool, walkFn util.WalkFunc) error { + return walkFn(path, nil, os.ErrPermission) + } + t.Cleanup(func() { + walk = origWalk + }) + + finder := newFS(log.NewTestLogger()) + paths, err := finder.getAbsPluginJSONPaths("test") + require.NoError(t, err) + require.Empty(t, paths) + }) + + t.Run("When scanning a folder that returns a non-handled error should return that error", func(t *testing.T) { + origWalk := walk + walk = func(path string, followSymlinks, detectSymlinkInfiniteLoop bool, walkFn util.WalkFunc) error { + return walkFn(path, nil, fmt.Errorf("random error")) + } + t.Cleanup(func() { + walk = origWalk + }) + + finder := newFS(log.NewTestLogger()) + paths, err := finder.getAbsPluginJSONPaths("test") + require.Error(t, err) + require.Empty(t, paths) + }) +} + +func TestFinder_validatePluginJSON(t *testing.T) { + type args struct { + data plugins.JSONData + } + tests := []struct { + name string + args args + err error + }{ + { + name: "Valid case", + args: args{ + data: plugins.JSONData{ + ID: "grafana-plugin-id", + Type: plugins.DataSource, + }, + }, + }, + { + name: "Invalid plugin ID", + args: args{ + data: plugins.JSONData{ + Type: plugins.Panel, + }, + }, + err: ErrInvalidPluginJSON, + }, + { + name: "Invalid plugin type", + args: args{ + data: plugins.JSONData{ + ID: "grafana-plugin-id", + Type: "test", + }, + }, + err: ErrInvalidPluginJSON, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := validatePluginJSON(tt.args.data); !errors.Is(err, tt.err) { + t.Errorf("validatePluginJSON() = %v, want %v", err, tt.err) + } + }) + } +} + +func TestFinder_readPluginJSON(t *testing.T) { + tests := []struct { + name string + pluginPath string + expected plugins.JSONData + failed bool + }{ + { + name: "Valid plugin", + pluginPath: "../../testdata/test-app/plugin.json", + expected: plugins.JSONData{ + ID: "test-app", + Type: "app", + Name: "Test App", + Info: plugins.Info{ + Author: plugins.InfoLink{ + Name: "Test Inc.", + URL: "http://test.com", + }, + Description: "Official Grafana Test App & Dashboard bundle", + Version: "1.0.0", + Links: []plugins.InfoLink{ + {Name: "Project site", URL: "http://project.com"}, + {Name: "License & Terms", URL: "http://license.com"}, + }, + Logos: plugins.Logos{ + Small: "img/logo_small.png", + Large: "img/logo_large.png", + }, + Screenshots: []plugins.Screenshots{ + {Path: "img/screenshot1.png", Name: "img1"}, + {Path: "img/screenshot2.png", Name: "img2"}, + }, + Updated: "2015-02-10", + }, + Dependencies: plugins.Dependencies{ + GrafanaVersion: "3.x.x", + Plugins: []plugins.Dependency{ + {Type: "datasource", ID: "graphite", Name: "Graphite", Version: "1.0.0"}, + {Type: "panel", ID: "graph", Name: "Graph", Version: "1.0.0"}, + }, + }, + Includes: []*plugins.Includes{ + {Name: "Nginx Connections", Path: "dashboards/connections.json", Type: "dashboard", Role: org.RoleViewer}, + {Name: "Nginx Memory", Path: "dashboards/memory.json", Type: "dashboard", Role: org.RoleViewer}, + {Name: "Nginx Panel", Type: "panel", Role: org.RoleViewer}, + {Name: "Nginx Datasource", Type: "datasource", Role: org.RoleViewer}, + }, + Backend: false, + }, + }, + { + name: "Invalid plugin JSON", + pluginPath: "../testdata/invalid-plugin-json/plugin.json", + failed: true, + }, + { + name: "Non-existing JSON file", + pluginPath: "nonExistingFile.json", + failed: true, + }, + } + + f := newFS(log.NewTestLogger()) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := f.readPluginJSON(tt.pluginPath) + if (err != nil) && !tt.failed { + t.Errorf("readPluginJSON() error = %v, failed %v", err, tt.failed) + return + } + if !cmp.Equal(got, tt.expected) { + t.Errorf("Unexpected pluginJSONData: %v", cmp.Diff(got, tt.expected)) + } + }) + } +} + +var localFSComparer = cmp.Comparer(func(fs1 plugins.LocalFS, fs2 plugins.LocalFS) bool { + fs1Files := fs1.Files() + fs2Files := fs2.Files() + + sort.SliceStable(fs1Files, func(i, j int) bool { + return fs1Files[i] < fs1Files[j] + }) + + sort.SliceStable(fs2Files, func(i, j int) bool { + return fs2Files[i] < fs2Files[j] + }) + + return cmp.Equal(fs1Files, fs2Files) && fs1.Base() == fs2.Base() +}) diff --git a/pkg/plugins/manager/loader/finder/ifaces.go b/pkg/plugins/manager/loader/finder/ifaces.go new file mode 100644 index 00000000000..cddbb7ecc91 --- /dev/null +++ b/pkg/plugins/manager/loader/finder/ifaces.go @@ -0,0 +1,11 @@ +package finder + +import ( + "context" + + "github.com/grafana/grafana/pkg/plugins" +) + +type Finder interface { + Find(ctx context.Context, uris ...string) ([]*plugins.FoundBundle, error) +} diff --git a/pkg/plugins/manager/loader/initializer/initializer_test.go b/pkg/plugins/manager/loader/initializer/initializer_test.go index d6a17b29f10..6cc505b274f 100644 --- a/pkg/plugins/manager/loader/initializer/initializer_test.go +++ b/pkg/plugins/manager/loader/initializer/initializer_test.go @@ -2,7 +2,6 @@ package initializer import ( "context" - "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -15,9 +14,6 @@ import ( ) func TestInitializer_Initialize(t *testing.T) { - absCurPath, err := filepath.Abs(".") - assert.NoError(t, err) - t.Run("core backend datasource", func(t *testing.T) { p := &plugins.Plugin{ JSONData: plugins.JSONData{ @@ -31,8 +27,7 @@ func TestInitializer_Initialize(t *testing.T) { }, Backend: true, }, - PluginDir: absCurPath, - Class: plugins.Core, + Class: plugins.Core, } i := &Initializer{ @@ -61,8 +56,7 @@ func TestInitializer_Initialize(t *testing.T) { }, Backend: true, }, - PluginDir: absCurPath, - Class: plugins.External, + Class: plugins.External, } i := &Initializer{ @@ -91,8 +85,7 @@ func TestInitializer_Initialize(t *testing.T) { }, Backend: true, }, - PluginDir: absCurPath, - Class: plugins.External, + Class: plugins.External, } i := &Initializer{ diff --git a/pkg/plugins/manager/loader/loader.go b/pkg/plugins/manager/loader/loader.go index 47ca1554c33..937091955a6 100644 --- a/pkg/plugins/manager/loader/loader.go +++ b/pkg/plugins/manager/loader/loader.go @@ -2,16 +2,11 @@ package loader import ( "context" - "encoding/json" "errors" "fmt" - "os" "path" - "path/filepath" - "runtime" "strings" - "github.com/grafana/grafana/pkg/infra/fs" "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/infra/slugify" "github.com/grafana/grafana/pkg/plugins" @@ -25,15 +20,9 @@ import ( "github.com/grafana/grafana/pkg/plugins/manager/signature" "github.com/grafana/grafana/pkg/plugins/pluginscdn" "github.com/grafana/grafana/pkg/plugins/storage" - "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/util" ) -var ( - ErrInvalidPluginJSON = errors.New("did not find valid type or id properties in plugin.json") - ErrInvalidPluginJSONFilePath = errors.New("invalid plugin.json filepath was provided") -) - var _ plugins.ErrorResolver = (*Loader)(nil) type Loader struct { @@ -64,7 +53,7 @@ func New(cfg *config.Cfg, license plugins.Licensing, authorizer plugins.PluginLo processManager process.Service, pluginStorage storage.Manager, roleRegistry plugins.RoleRegistry, pluginsCDNService *pluginscdn.Service, assetPath *assetpath.Service) *Loader { return &Loader{ - pluginFinder: finder.New(), + pluginFinder: finder.NewService(), pluginRegistry: pluginRegistry, pluginInitializer: initializer.New(cfg, backendProvider, license), signatureValidator: signature.NewValidator(authorizer), @@ -80,95 +69,65 @@ func New(cfg *config.Cfg, license plugins.Licensing, authorizer plugins.PluginLo } func (l *Loader) Load(ctx context.Context, class plugins.Class, paths []string) ([]*plugins.Plugin, error) { - pluginJSONPaths, err := l.pluginFinder.Find(paths) + found, err := l.pluginFinder.Find(ctx, paths...) if err != nil { return nil, err } - return l.loadPlugins(ctx, class, pluginJSONPaths) + return l.loadPlugins(ctx, class, found) } -func (l *Loader) createPluginsForLoading(class plugins.Class, foundPlugins foundPlugins) map[string]*plugins.Plugin { - loadedPlugins := make(map[string]*plugins.Plugin) - for pluginDir, pluginJSON := range foundPlugins { - plugin, err := l.createPluginBase(pluginJSON, class, pluginDir) - if err != nil { - l.log.Warn("Could not create plugin base", "pluginID", pluginJSON.ID, "err", err) +func (l *Loader) loadPlugins(ctx context.Context, class plugins.Class, found []*plugins.FoundBundle) ([]*plugins.Plugin, error) { + var loadedPlugins []*plugins.Plugin + for _, p := range found { + if _, exists := l.pluginRegistry.Plugin(ctx, p.Primary.JSONData.ID); exists { + l.log.Warn("Skipping plugin loading as it's a duplicate", "pluginID", p.Primary.JSONData.ID) continue } - // calculate initial signature state var sig plugins.Signature - if l.pluginsCDN.PluginSupported(plugin.ID) { + if l.pluginsCDN.PluginSupported(p.Primary.JSONData.ID) { // CDN plugins have no signature checks for now. sig = plugins.Signature{Status: plugins.SignatureValid} } else { - sig, err = signature.Calculate(l.log, plugin) + var err error + sig, err = signature.Calculate(l.log, class, p.Primary) if err != nil { - l.log.Warn("Could not calculate plugin signature state", "pluginID", plugin.ID, "err", err) + l.log.Warn("Could not calculate plugin signature state", "pluginID", p.Primary.JSONData.ID, "err", err) continue } } + plugin, err := l.createPluginBase(p.Primary.JSONData, class, p.Primary.FS) + if err != nil { + l.log.Error("Could not create primary plugin base", "pluginID", p.Primary.JSONData.ID, "err", err) + continue + } + plugin.Signature = sig.Status plugin.SignatureType = sig.Type plugin.SignatureOrg = sig.SigningOrg - loadedPlugins[plugin.PluginDir] = plugin - } - return loadedPlugins -} + loadedPlugins = append(loadedPlugins, plugin) -func (l *Loader) loadPlugins(ctx context.Context, class plugins.Class, pluginJSONPaths []string) ([]*plugins.Plugin, error) { - var foundPlugins = foundPlugins{} - - // load plugin.json files and map directory to JSON data - for _, pluginJSONPath := range pluginJSONPaths { - plugin, err := l.readPluginJSON(pluginJSONPath) - if err != nil { - l.log.Warn("Skipping plugin loading as its plugin.json could not be read", "path", pluginJSONPath, "err", err) - continue - } - - pluginJSONAbsPath, err := filepath.Abs(pluginJSONPath) - if err != nil { - l.log.Warn("Skipping plugin loading as absolute plugin.json path could not be calculated", "pluginID", plugin.ID, "err", err) - continue - } - - if _, dupe := foundPlugins[filepath.Dir(pluginJSONAbsPath)]; dupe { - l.log.Warn("Skipping plugin loading as it's a duplicate", "pluginID", plugin.ID) - continue - } - foundPlugins[filepath.Dir(pluginJSONAbsPath)] = plugin - } - - // get all registered plugins - registeredPlugins := make(map[string]struct{}) - for _, p := range l.pluginRegistry.Plugins(ctx) { - registeredPlugins[p.ID] = struct{}{} - } - - foundPlugins.stripDuplicates(registeredPlugins, l.log) - - // create plugins structs and calculate signatures - loadedPlugins := l.createPluginsForLoading(class, foundPlugins) - - // wire up plugin dependencies - for _, plugin := range loadedPlugins { - ancestors := strings.Split(plugin.PluginDir, string(filepath.Separator)) - ancestors = ancestors[0 : len(ancestors)-1] - pluginPath := "" - - if runtime.GOOS != "windows" && filepath.IsAbs(plugin.PluginDir) { - pluginPath = "/" - } - for _, ancestor := range ancestors { - pluginPath = filepath.Join(pluginPath, ancestor) - if parentPlugin, ok := loadedPlugins[pluginPath]; ok { - plugin.Parent = parentPlugin - plugin.Parent.Children = append(plugin.Parent.Children, plugin) - break + for _, c := range p.Children { + if _, exists := l.pluginRegistry.Plugin(ctx, c.JSONData.ID); exists { + l.log.Warn("Skipping plugin loading as it's a duplicate", "pluginID", p.Primary.JSONData.ID) + continue } + + cp, err := l.createPluginBase(c.JSONData, class, c.FS) + if err != nil { + l.log.Error("Could not create child plugin base", "pluginID", p.Primary.JSONData.ID, "err", err) + continue + } + cp.Parent = plugin + cp.Signature = sig.Status + cp.SignatureType = sig.Type + cp.SignatureOrg = sig.SigningOrg + + plugin.Children = append(plugin.Children, cp) + + loadedPlugins = append(loadedPlugins, cp) } } @@ -191,14 +150,12 @@ func (l *Loader) loadPlugins(ctx context.Context, class plugins.Class, pluginJSO // verify module.js exists for SystemJS to load. // CDN plugins can be loaded with plugin.json only, so do not warn for those. if !plugin.IsRenderer() && !plugin.IsCorePlugin() { - module := filepath.Join(plugin.PluginDir, "module.js") - if exists, err := fs.Exists(module); err != nil { - return nil, err - } else if !exists && !l.pluginsCDN.PluginSupported(plugin.ID) { - l.log.Warn("Plugin missing module.js", - "pluginID", plugin.ID, - "warning", "Missing module.js, If you loaded this plugin from git, make sure to compile it.", - "path", module) + _, err := plugin.FS.Open("module.js") + if err != nil { + if errors.Is(err, plugins.ErrFileNotExist) && !l.pluginsCDN.PluginSupported(plugin.ID) { + l.log.Warn("Plugin missing module.js", "pluginID", plugin.ID, + "warning", "Missing module.js, If you loaded this plugin from git, make sure to compile it.") + } } } @@ -221,7 +178,7 @@ func (l *Loader) loadPlugins(ctx context.Context, class plugins.Class, pluginJSO metrics.SetPluginBuildInformation(p.ID, string(p.Type), p.Info.Version, string(p.Signature)) if errDeclareRoles := l.roleRegistry.DeclarePluginRoles(ctx, p.ID, p.Name, p.Roles); errDeclareRoles != nil { - l.log.Warn("Declare plugin roles failed.", "pluginID", p.ID, "path", p.PluginDir, "error", errDeclareRoles) + l.log.Warn("Declare plugin roles failed.", "pluginID", p.ID, "err", errDeclareRoles) } } @@ -260,7 +217,7 @@ func (l *Loader) load(ctx context.Context, p *plugins.Plugin) error { } if p.IsExternalPlugin() { - if err := l.pluginStorage.Register(ctx, p.ID, p.PluginDir); err != nil { + if err := l.pluginStorage.Register(ctx, p.ID, p.FS.Base()); err != nil { return err } } @@ -271,7 +228,6 @@ func (l *Loader) load(ctx context.Context, p *plugins.Plugin) error { func (l *Loader) unload(ctx context.Context, p *plugins.Plugin) error { l.log.Debug("Stopping plugin process", "pluginId", p.ID) - // TODO confirm the sequence of events is safe if err := l.processManager.Stop(ctx, p.ID); err != nil { return err } @@ -287,70 +243,21 @@ func (l *Loader) unload(ctx context.Context, p *plugins.Plugin) error { return nil } -func (l *Loader) readPluginJSON(pluginJSONPath string) (plugins.JSONData, error) { - l.log.Debug("Loading plugin", "path", pluginJSONPath) - - if !strings.EqualFold(filepath.Ext(pluginJSONPath), ".json") { - return plugins.JSONData{}, ErrInvalidPluginJSONFilePath - } - - // nolint:gosec - // We can ignore the gosec G304 warning on this one because `currentPath` is based - // on plugin the folder structure on disk and not user input. - reader, err := os.Open(pluginJSONPath) - if err != nil { - return plugins.JSONData{}, err - } - - plugin := plugins.JSONData{} - if err = json.NewDecoder(reader).Decode(&plugin); err != nil { - return plugins.JSONData{}, err - } - - if err = reader.Close(); err != nil { - l.log.Warn("Failed to close JSON file", "path", pluginJSONPath, "err", err) - } - - if err = validatePluginJSON(plugin); err != nil { - return plugins.JSONData{}, err - } - - if plugin.ID == "grafana-piechart-panel" { - plugin.Name = "Pie Chart (old)" - } - - if len(plugin.Dependencies.Plugins) == 0 { - plugin.Dependencies.Plugins = []plugins.Dependency{} - } - - if plugin.Dependencies.GrafanaVersion == "" { - plugin.Dependencies.GrafanaVersion = "*" - } - - for _, include := range plugin.Includes { - if include.Role == "" { - include.Role = org.RoleViewer - } - } - - return plugin, nil -} - -func (l *Loader) createPluginBase(pluginJSON plugins.JSONData, class plugins.Class, pluginDir string) (*plugins.Plugin, error) { - baseURL, err := l.assetPath.Base(pluginJSON, class, pluginDir) +func (l *Loader) createPluginBase(pluginJSON plugins.JSONData, class plugins.Class, files plugins.FS) (*plugins.Plugin, error) { + baseURL, err := l.assetPath.Base(pluginJSON, class, files.Base()) if err != nil { return nil, fmt.Errorf("base url: %w", err) } - moduleURL, err := l.assetPath.Module(pluginJSON, class, pluginDir) + moduleURL, err := l.assetPath.Module(pluginJSON, class, files.Base()) if err != nil { return nil, fmt.Errorf("module url: %w", err) } plugin := &plugins.Plugin{ - JSONData: pluginJSON, - PluginDir: pluginDir, - BaseURL: baseURL, - Module: moduleURL, - Class: class, + JSONData: pluginJSON, + FS: files, + BaseURL: baseURL, + Module: moduleURL, + Class: class, } plugin.SetLogger(log.New(fmt.Sprintf("plugin.%s", plugin.ID))) @@ -409,7 +316,7 @@ func configureAppChildPlugin(parent *plugins.Plugin, child *plugins.Plugin) { if !parent.IsApp() { return } - appSubPath := strings.ReplaceAll(strings.Replace(child.PluginDir, parent.PluginDir, "", 1), "\\", "/") + appSubPath := strings.ReplaceAll(strings.Replace(child.FS.Base(), parent.FS.Base(), "", 1), "\\", "/") child.IncludedInAppID = parent.ID child.BaseURL = parent.BaseURL @@ -435,26 +342,3 @@ func (l *Loader) PluginErrors() []*plugins.Error { return errs } - -func validatePluginJSON(data plugins.JSONData) error { - if data.ID == "" || !data.Type.IsValid() { - return ErrInvalidPluginJSON - } - return nil -} - -type foundPlugins map[string]plugins.JSONData - -// stripDuplicates will strip duplicate plugins or plugins that already exist -func (f *foundPlugins) stripDuplicates(existingPlugins map[string]struct{}, log log.Logger) { - pluginsByID := make(map[string]struct{}) - for k, scannedPlugin := range *f { - if _, existing := existingPlugins[scannedPlugin.ID]; existing { - log.Debug("Skipping plugin as it's already installed", "plugin", scannedPlugin.ID) - delete(*f, k) - continue - } - - pluginsByID[scannedPlugin.ID] = struct{}{} - } -} diff --git a/pkg/plugins/manager/loader/loader_test.go b/pkg/plugins/manager/loader/loader_test.go index 3408a3d20dd..887b381bcba 100644 --- a/pkg/plugins/manager/loader/loader_test.go +++ b/pkg/plugins/manager/loader/loader_test.go @@ -2,7 +2,7 @@ package loader import ( "context" - "errors" + "os" "path/filepath" "sort" "testing" @@ -20,11 +20,25 @@ import ( "github.com/grafana/grafana/pkg/plugins/manager/fakes" "github.com/grafana/grafana/pkg/plugins/manager/loader/initializer" "github.com/grafana/grafana/pkg/plugins/manager/signature" - "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/setting" ) -var compareOpts = cmpopts.IgnoreFields(plugins.Plugin{}, "client", "log") +var compareOpts = []cmp.Option{cmpopts.IgnoreFields(plugins.Plugin{}, "client", "log"), localFSComparer} + +var localFSComparer = cmp.Comparer(func(fs1 plugins.LocalFS, fs2 plugins.LocalFS) bool { + fs1Files := fs1.Files() + fs2Files := fs2.Files() + + sort.SliceStable(fs1Files, func(i, j int) bool { + return fs1Files[i] < fs1Files[j] + }) + + sort.SliceStable(fs2Files, func(i, j int) bool { + return fs2Files[i] < fs2Files[j] + }) + + return cmp.Equal(fs1Files, fs2Files) && fs1.Base() == fs2.Base() +}) func TestLoader_Load(t *testing.T) { corePluginDir, err := filepath.Abs("./../../../../public") @@ -86,9 +100,11 @@ func TestLoader_Load(t *testing.T) { Backend: true, QueryOptions: map[string]bool{"minInterval": true}, }, - Module: "app/plugins/datasource/cloudwatch/module", - BaseURL: "public/app/plugins/datasource/cloudwatch", - PluginDir: filepath.Join(corePluginDir, "app/plugins/datasource/cloudwatch"), + Module: "app/plugins/datasource/cloudwatch/module", + BaseURL: "public/app/plugins/datasource/cloudwatch", + FS: plugins.NewLocalFS( + filesInDir(t, filepath.Join(corePluginDir, "app/plugins/datasource/cloudwatch")), + filepath.Join(corePluginDir, "app/plugins/datasource/cloudwatch")), Signature: plugins.SignatureInternal, Class: plugins.Core, }, @@ -125,9 +141,12 @@ func TestLoader_Load(t *testing.T) { Backend: true, State: "alpha", }, - Module: "plugins/test-datasource/module", - BaseURL: "public/plugins/test-datasource", - PluginDir: filepath.Join(parentDir, "testdata/valid-v2-signature/plugin/"), + Module: "plugins/test-datasource/module", + BaseURL: "public/plugins/test-datasource", + FS: plugins.NewLocalFS( + filesInDir(t, filepath.Join(parentDir, "testdata/valid-v2-signature/plugin/")), + filepath.Join(parentDir, "testdata/valid-v2-signature/plugin/"), + ), Signature: "valid", SignatureType: plugins.GrafanaSignature, SignatureOrg: "Grafana Labs", @@ -201,10 +220,20 @@ func TestLoader_Load(t *testing.T) { }, }, }, - Class: plugins.External, - Module: "plugins/test-app/module", - BaseURL: "public/plugins/test-app", - PluginDir: filepath.Join(parentDir, "testdata/includes-symlinks"), + Class: plugins.External, + Module: "plugins/test-app/module", + BaseURL: "public/plugins/test-app", + FS: plugins.NewLocalFS( + map[string]struct{}{ + filepath.Join(parentDir, "testdata/includes-symlinks", "/MANIFEST.txt"): {}, + filepath.Join(parentDir, "testdata/includes-symlinks", "dashboards/connections.json"): {}, + filepath.Join(parentDir, "testdata/includes-symlinks", "dashboards/extra/memory.json"): {}, + filepath.Join(parentDir, "testdata/includes-symlinks", "plugin.json"): {}, + filepath.Join(parentDir, "testdata/includes-symlinks", "symlink_to_txt"): {}, + filepath.Join(parentDir, "testdata/includes-symlinks", "text.txt"): {}, + }, + filepath.Join(parentDir, "testdata/includes-symlinks"), + ), Signature: "valid", SignatureType: plugins.GrafanaSignature, SignatureOrg: "Grafana Labs", @@ -241,10 +270,13 @@ func TestLoader_Load(t *testing.T) { Backend: true, State: plugins.AlphaRelease, }, - Class: plugins.External, - Module: "plugins/test-datasource/module", - BaseURL: "public/plugins/test-datasource", - PluginDir: filepath.Join(parentDir, "testdata/unsigned-datasource/plugin"), + Class: plugins.External, + Module: "plugins/test-datasource/module", + BaseURL: "public/plugins/test-datasource", + FS: plugins.NewLocalFS( + filesInDir(t, filepath.Join(parentDir, "testdata/unsigned-datasource/plugin")), + filepath.Join(parentDir, "testdata/unsigned-datasource/plugin"), + ), Signature: "unsigned", }, }, @@ -292,10 +324,13 @@ func TestLoader_Load(t *testing.T) { Backend: true, State: plugins.AlphaRelease, }, - Class: plugins.External, - Module: "plugins/test-datasource/module", - BaseURL: "public/plugins/test-datasource", - PluginDir: filepath.Join(parentDir, "testdata/unsigned-datasource/plugin"), + Class: plugins.External, + Module: "plugins/test-datasource/module", + BaseURL: "public/plugins/test-datasource", + FS: plugins.NewLocalFS( + filesInDir(t, filepath.Join(parentDir, "testdata/unsigned-datasource/plugin")), + filepath.Join(parentDir, "testdata/unsigned-datasource/plugin"), + ), Signature: plugins.SignatureUnsigned, }, }, @@ -399,11 +434,14 @@ func TestLoader_Load(t *testing.T) { Backend: false, }, DefaultNavURL: "/plugins/test-app/page/root-page-react", - PluginDir: filepath.Join(parentDir, "testdata/test-app-with-includes"), - Class: plugins.External, - Signature: plugins.SignatureUnsigned, - Module: "plugins/test-app/module", - BaseURL: "public/plugins/test-app", + FS: plugins.NewLocalFS(map[string]struct{}{ + filepath.Join(parentDir, "testdata/test-app-with-includes", "dashboards/memory.json"): {}, + filepath.Join(parentDir, "testdata/test-app-with-includes", "plugin.json"): {}, + }, filepath.Join(parentDir, "testdata/test-app-with-includes")), + Class: plugins.External, + Signature: plugins.SignatureUnsigned, + Module: "plugins/test-app/module", + BaseURL: "public/plugins/test-app", }, }, }, @@ -454,7 +492,9 @@ func TestLoader_Load(t *testing.T) { Plugins: []plugins.Dependency{}, }, }, - PluginDir: filepath.Join(parentDir, "testdata/cdn/plugin"), + FS: plugins.NewLocalFS(map[string]struct{}{ + filepath.Join(parentDir, "testdata/cdn/plugin", "plugin.json"): {}, + }, filepath.Join(parentDir, "testdata/cdn/plugin")), Class: plugins.External, Signature: plugins.SignatureValid, BaseURL: "plugin-cdn/grafana-worldmap-panel/0.3.3/public/plugins/grafana-worldmap-panel", @@ -478,8 +518,8 @@ func TestLoader_Load(t *testing.T) { t.Run(tt.name, func(t *testing.T) { got, err := l.Load(context.Background(), tt.class, tt.pluginPaths) require.NoError(t, err) - if !cmp.Equal(got, tt.want, compareOpts) { - t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, tt.want, compareOpts)) + if !cmp.Equal(got, tt.want, compareOpts...) { + t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, tt.want, compareOpts...)) } pluginErrs := l.PluginErrors() @@ -600,10 +640,13 @@ func TestLoader_Load_MultiplePlugins(t *testing.T) { Executable: "test", State: plugins.AlphaRelease, }, - Class: plugins.External, - Module: "plugins/test-datasource/module", - BaseURL: "public/plugins/test-datasource", - PluginDir: filepath.Join(parentDir, "testdata/valid-v2-pvt-signature/plugin"), + Class: plugins.External, + Module: "plugins/test-datasource/module", + BaseURL: "public/plugins/test-datasource", + FS: plugins.NewLocalFS(map[string]struct{}{ + filepath.Join(parentDir, "testdata/valid-v2-pvt-signature/plugin/plugin.json"): {}, + filepath.Join(parentDir, "testdata/valid-v2-pvt-signature/plugin/MANIFEST.txt"): {}, + }, filepath.Join(parentDir, "testdata/valid-v2-pvt-signature/plugin")), Signature: "valid", SignatureType: plugins.PrivateSignature, SignatureOrg: "Will Browne", @@ -641,8 +684,8 @@ func TestLoader_Load_MultiplePlugins(t *testing.T) { sort.SliceStable(got, func(i, j int) bool { return got[i].ID < got[j].ID }) - if !cmp.Equal(got, tt.want, compareOpts) { - t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, tt.want, compareOpts)) + if !cmp.Equal(got, tt.want, compareOpts...) { + t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, tt.want, compareOpts...)) } pluginErrs := l.PluginErrors() require.Equal(t, len(tt.pluginErrors), len(pluginErrs)) @@ -717,7 +760,10 @@ func TestLoader_Load_RBACReady(t *testing.T) { }, Backend: false, }, - PluginDir: pluginDir, + FS: plugins.NewLocalFS(map[string]struct{}{ + filepath.Join(pluginDir, "plugin.json"): {}, + filepath.Join(pluginDir, "MANIFEST.txt"): {}, + }, pluginDir), Class: plugins.External, Signature: plugins.SignatureValid, SignatureType: plugins.PrivateSignature, @@ -749,8 +795,8 @@ func TestLoader_Load_RBACReady(t *testing.T) { got, err := l.Load(context.Background(), plugins.External, tt.pluginPaths) require.NoError(t, err) - if !cmp.Equal(got, tt.want, compareOpts) { - t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, tt.want, compareOpts)) + if !cmp.Equal(got, tt.want, compareOpts...) { + t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, tt.want, compareOpts...)) } pluginErrs := l.PluginErrors() require.Len(t, pluginErrs, 0) @@ -799,7 +845,10 @@ func TestLoader_Load_Signature_RootURL(t *testing.T) { Backend: true, Executable: "test", }, - PluginDir: filepath.Join(parentDir, "/testdata/valid-v2-pvt-signature-root-url-uri/plugin"), + FS: plugins.NewLocalFS(map[string]struct{}{ + filepath.Join(filepath.Join(parentDir, "/testdata/valid-v2-pvt-signature-root-url-uri/plugin"), "plugin.json"): {}, + filepath.Join(filepath.Join(parentDir, "/testdata/valid-v2-pvt-signature-root-url-uri/plugin"), "MANIFEST.txt"): {}, + }, filepath.Join(parentDir, "/testdata/valid-v2-pvt-signature-root-url-uri/plugin")), Class: plugins.External, Signature: plugins.SignatureValid, SignatureType: plugins.PrivateSignature, @@ -822,8 +871,8 @@ func TestLoader_Load_Signature_RootURL(t *testing.T) { got, err := l.Load(context.Background(), plugins.External, paths) require.NoError(t, err) - if !cmp.Equal(got, expected, compareOpts) { - t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts)) + if !cmp.Equal(got, expected, compareOpts...) { + t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts...)) } verifyState(t, expected, reg, procPrvdr, storage, procMgr) }) @@ -878,7 +927,7 @@ func TestLoader_Load_DuplicatePlugins(t *testing.T) { }, Backend: false, }, - PluginDir: pluginDir, + FS: plugins.NewLocalFS(filesInDir(t, pluginDir), pluginDir), Class: plugins.External, Signature: plugins.SignatureValid, SignatureType: plugins.GrafanaSignature, @@ -901,8 +950,8 @@ func TestLoader_Load_DuplicatePlugins(t *testing.T) { got, err := l.Load(context.Background(), plugins.External, []string{pluginDir, pluginDir}) require.NoError(t, err) - if !cmp.Equal(got, expected, compareOpts) { - t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts)) + if !cmp.Equal(got, expected, compareOpts...) { + t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts...)) } verifyState(t, expected, reg, procPrvdr, storage, procMgr) @@ -939,9 +988,10 @@ func TestLoader_Load_NestedPlugins(t *testing.T) { }, Backend: true, }, - Module: "plugins/test-datasource/module", - BaseURL: "public/plugins/test-datasource", - PluginDir: filepath.Join(rootDir, "testdata/nested-plugins/parent"), + Module: "plugins/test-datasource/module", + BaseURL: "public/plugins/test-datasource", + FS: plugins.NewLocalFS(filesInDir(t, filepath.Join(rootDir, "testdata/nested-plugins/parent")), + filepath.Join(rootDir, "testdata/nested-plugins/parent")), Signature: plugins.SignatureValid, SignatureType: plugins.GrafanaSignature, SignatureOrg: "Grafana Labs", @@ -971,9 +1021,10 @@ func TestLoader_Load_NestedPlugins(t *testing.T) { Plugins: []plugins.Dependency{}, }, }, - Module: "plugins/test-panel/module", - BaseURL: "public/plugins/test-panel", - PluginDir: filepath.Join(rootDir, "testdata/nested-plugins/parent/nested"), + Module: "plugins/test-panel/module", + BaseURL: "public/plugins/test-panel", + FS: plugins.NewLocalFS(filesInDir(t, filepath.Join(rootDir, "testdata/nested-plugins/parent/nested")), + filepath.Join(rootDir, "testdata/nested-plugins/parent/nested")), Signature: plugins.SignatureValid, SignatureType: plugins.GrafanaSignature, SignatureOrg: "Grafana Labs", @@ -1004,8 +1055,8 @@ func TestLoader_Load_NestedPlugins(t *testing.T) { }) expected := []*plugins.Plugin{parent, child} - if !cmp.Equal(got, expected, compareOpts) { - t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts)) + if !cmp.Equal(got, expected, compareOpts...) { + t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts...)) } verifyState(t, expected, reg, procPrvdr, storage, procMgr) @@ -1019,8 +1070,8 @@ func TestLoader_Load_NestedPlugins(t *testing.T) { return got[i].ID < got[j].ID }) - if !cmp.Equal(got, []*plugins.Plugin{}, compareOpts) { - t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts)) + if !cmp.Equal(got, []*plugins.Plugin{}, compareOpts...) { + t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts...)) } verifyState(t, expected, reg, procPrvdr, storage, procMgr) @@ -1098,9 +1149,10 @@ func TestLoader_Load_NestedPlugins(t *testing.T) { }, Backend: false, }, - Module: "plugins/myorgid-simple-app/module", - BaseURL: "public/plugins/myorgid-simple-app", - PluginDir: filepath.Join(rootDir, "testdata/app-with-child/dist"), + Module: "plugins/myorgid-simple-app/module", + BaseURL: "public/plugins/myorgid-simple-app", + FS: plugins.NewLocalFS(filesInDir(t, filepath.Join(rootDir, "testdata/app-with-child/dist")), + filepath.Join(rootDir, "testdata/app-with-child/dist")), DefaultNavURL: "/plugins/myorgid-simple-app/page/root-page-react", Signature: plugins.SignatureValid, SignatureType: plugins.GrafanaSignature, @@ -1136,9 +1188,10 @@ func TestLoader_Load_NestedPlugins(t *testing.T) { Plugins: []plugins.Dependency{}, }, }, - Module: "plugins/myorgid-simple-app/child/module", - BaseURL: "public/plugins/myorgid-simple-app", - PluginDir: filepath.Join(rootDir, "testdata/app-with-child/dist/child"), + Module: "plugins/myorgid-simple-app/child/module", + BaseURL: "public/plugins/myorgid-simple-app", + FS: plugins.NewLocalFS(filesInDir(t, filepath.Join(rootDir, "testdata/app-with-child/dist/child")), + filepath.Join(rootDir, "testdata/app-with-child/dist/child")), IncludedInAppID: parent.ID, Signature: plugins.SignatureValid, SignatureType: plugins.GrafanaSignature, @@ -1168,205 +1221,27 @@ func TestLoader_Load_NestedPlugins(t *testing.T) { return got[i].ID < got[j].ID }) - if !cmp.Equal(got, expected, compareOpts) { - t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts)) + if !cmp.Equal(got, expected, compareOpts...) { + t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts...)) } verifyState(t, expected, reg, procPrvdr, storage, procMgr) - - t.Run("order of loaded parent and child plugins gives same output", func(t *testing.T) { - parentPluginJSON := filepath.Join(rootDir, "testdata/app-with-child/dist/plugin.json") - childPluginJSON := filepath.Join(rootDir, "testdata/app-with-child/dist/child/plugin.json") - - reg = fakes.NewFakePluginRegistry() - storage = fakes.NewFakePluginStorage() - procPrvdr = fakes.NewFakeBackendProcessProvider() - procMgr = fakes.NewFakeProcessManager() - l = newLoader(&config.Cfg{}, func(l *Loader) { - l.pluginRegistry = reg - l.pluginStorage = storage - l.processManager = procMgr - l.pluginInitializer = initializer.New(&config.Cfg{}, procPrvdr, fakes.NewFakeLicensingService()) - }) - got, err = l.loadPlugins(context.Background(), plugins.External, []string{parentPluginJSON, childPluginJSON}) - require.NoError(t, err) - - // to ensure we can compare with expected - sort.SliceStable(got, func(i, j int) bool { - return got[i].ID < got[j].ID - }) - - if !cmp.Equal(got, expected, compareOpts) { - t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts)) - } - - verifyState(t, expected, reg, procPrvdr, storage, procMgr) - - reg = fakes.NewFakePluginRegistry() - storage = fakes.NewFakePluginStorage() - procPrvdr = fakes.NewFakeBackendProcessProvider() - procMgr = fakes.NewFakeProcessManager() - l = newLoader(&config.Cfg{}, func(l *Loader) { - l.pluginRegistry = reg - l.pluginStorage = storage - l.processManager = procMgr - l.pluginInitializer = initializer.New(&config.Cfg{}, procPrvdr, fakes.NewFakeLicensingService()) - }) - got, err = l.loadPlugins(context.Background(), plugins.External, []string{childPluginJSON, parentPluginJSON}) - require.NoError(t, err) - - // to ensure we can compare with expected - sort.SliceStable(got, func(i, j int) bool { - return got[i].ID < got[j].ID - }) - - if !cmp.Equal(got, expected, compareOpts) { - t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts)) - } - - verifyState(t, expected, reg, procPrvdr, storage, procMgr) - }) }) } -func TestLoader_readPluginJSON(t *testing.T) { - tests := []struct { - name string - pluginPath string - expected plugins.JSONData - failed bool - }{ - { - name: "Valid plugin", - pluginPath: "../testdata/test-app/plugin.json", - expected: plugins.JSONData{ - ID: "test-app", - Type: "app", - Name: "Test App", - Info: plugins.Info{ - Author: plugins.InfoLink{ - Name: "Test Inc.", - URL: "http://test.com", - }, - Description: "Official Grafana Test App & Dashboard bundle", - Version: "1.0.0", - Links: []plugins.InfoLink{ - {Name: "Project site", URL: "http://project.com"}, - {Name: "License & Terms", URL: "http://license.com"}, - }, - Logos: plugins.Logos{ - Small: "img/logo_small.png", - Large: "img/logo_large.png", - }, - Screenshots: []plugins.Screenshots{ - {Path: "img/screenshot1.png", Name: "img1"}, - {Path: "img/screenshot2.png", Name: "img2"}, - }, - Updated: "2015-02-10", - }, - Dependencies: plugins.Dependencies{ - GrafanaVersion: "3.x.x", - Plugins: []plugins.Dependency{ - {Type: "datasource", ID: "graphite", Name: "Graphite", Version: "1.0.0"}, - {Type: "panel", ID: "graph", Name: "Graph", Version: "1.0.0"}, - }, - }, - Includes: []*plugins.Includes{ - {Name: "Nginx Connections", Path: "dashboards/connections.json", Type: "dashboard", Role: org.RoleViewer}, - {Name: "Nginx Memory", Path: "dashboards/memory.json", Type: "dashboard", Role: org.RoleViewer}, - {Name: "Nginx Panel", Type: "panel", Role: org.RoleViewer}, - {Name: "Nginx Datasource", Type: "datasource", Role: org.RoleViewer}, - }, - Backend: false, - }, - }, - { - name: "Invalid plugin JSON", - pluginPath: "../testdata/invalid-plugin-json/plugin.json", - failed: true, - }, - { - name: "Non-existing JSON file", - pluginPath: "nonExistingFile.json", - failed: true, - }, - } - - l := newLoader(nil) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := l.readPluginJSON(tt.pluginPath) - if (err != nil) && !tt.failed { - t.Errorf("readPluginJSON() error = %v, failed %v", err, tt.failed) - return - } - if !cmp.Equal(got, tt.expected, compareOpts) { - t.Errorf("Unexpected pluginJSONData: %v", cmp.Diff(got, tt.expected, compareOpts)) - } - }) - } -} - -func Test_validatePluginJSON(t *testing.T) { - type args struct { - data plugins.JSONData - } - tests := []struct { - name string - args args - err error - }{ - { - name: "Valid case", - args: args{ - data: plugins.JSONData{ - ID: "grafana-plugin-id", - Type: plugins.DataSource, - }, - }, - }, - { - name: "Invalid plugin ID", - args: args{ - data: plugins.JSONData{ - Type: plugins.Panel, - }, - }, - err: ErrInvalidPluginJSON, - }, - { - name: "Invalid plugin type", - args: args{ - data: plugins.JSONData{ - ID: "grafana-plugin-id", - Type: "test", - }, - }, - err: ErrInvalidPluginJSON, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := validatePluginJSON(tt.args.data); !errors.Is(err, tt.err) { - t.Errorf("validatePluginJSON() = %v, want %v", err, tt.err) - } - }) - } -} - func Test_setPathsBasedOnApp(t *testing.T) { t.Run("When setting paths based on core plugin on Windows", func(t *testing.T) { child := &plugins.Plugin{ - PluginDir: "c:\\grafana\\public\\app\\plugins\\app\\testdata-app\\datasources\\datasource", + FS: fakes.NewFakePluginFiles("c:\\grafana\\public\\app\\plugins\\app\\testdata-app\\datasources\\datasource"), } parent := &plugins.Plugin{ JSONData: plugins.JSONData{ Type: plugins.App, ID: "testdata-app", }, - Class: plugins.Core, - PluginDir: "c:\\grafana\\public\\app\\plugins\\app\\testdata-app", - BaseURL: "public/app/plugins/app/testdata-app", + Class: plugins.Core, + FS: fakes.NewFakePluginFiles("c:\\grafana\\public\\app\\plugins\\app\\testdata-app"), + BaseURL: "public/app/plugins/app/testdata-app", } configureAppChildPlugin(parent, child) @@ -1395,8 +1270,8 @@ func verifyState(t *testing.T, ps []*plugins.Plugin, reg *fakes.FakePluginRegist t.Helper() for _, p := range ps { - if !cmp.Equal(p, reg.Store[p.ID], compareOpts) { - t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(p, reg.Store[p.ID], compareOpts)) + if !cmp.Equal(p, reg.Store[p.ID], compareOpts...) { + t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(p, reg.Store[p.ID], compareOpts...)) } if p.Backend { @@ -1418,3 +1293,40 @@ func verifyState(t *testing.T, ps []*plugins.Plugin, reg *fakes.FakePluginRegist require.Zero(t, procMngr.Stopped[p.ID]) } } + +func filesInDir(t *testing.T, dir string) map[string]struct{} { + files, err := collectFilesWithin(dir) + if err != nil { + t.Logf("Could not collect plugin file info. Err: %v", err) + return map[string]struct{}{} + } + return files +} + +func collectFilesWithin(dir string) (map[string]struct{}, error) { + files := map[string]struct{}{} + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // skip directories + if info.IsDir() { + return nil + } + + // verify that file is within plugin directory + //file, err := filepath.Rel(dir, path) + //if err != nil { + // return err + //} + //if strings.HasPrefix(file, ".."+string(filepath.Separator)) { + // return fmt.Errorf("file '%s' not inside of plugin directory", file) + //} + + files[path] = struct{}{} + return nil + }) + + return files, err +} diff --git a/pkg/plugins/manager/manager_integration_test.go b/pkg/plugins/manager/manager_integration_test.go index acd410f8568..37280ffc030 100644 --- a/pkg/plugins/manager/manager_integration_test.go +++ b/pkg/plugins/manager/manager_integration_test.go @@ -4,13 +4,9 @@ import ( "context" "encoding/json" "path/filepath" - "strings" "testing" "time" - "github.com/grafana/grafana/pkg/plugins/manager/loader/assetpath" - "github.com/grafana/grafana/pkg/plugins/pluginscdn" - "github.com/grafana/grafana-azure-sdk-go/azsettings" "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" @@ -27,10 +23,12 @@ import ( "github.com/grafana/grafana/pkg/plugins/manager/client" "github.com/grafana/grafana/pkg/plugins/manager/fakes" "github.com/grafana/grafana/pkg/plugins/manager/loader" + "github.com/grafana/grafana/pkg/plugins/manager/loader/assetpath" "github.com/grafana/grafana/pkg/plugins/manager/registry" "github.com/grafana/grafana/pkg/plugins/manager/signature" "github.com/grafana/grafana/pkg/plugins/manager/sources" "github.com/grafana/grafana/pkg/plugins/manager/store" + "github.com/grafana/grafana/pkg/plugins/pluginscdn" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/licensing" "github.com/grafana/grafana/pkg/services/searchV2" @@ -126,7 +124,7 @@ func TestIntegrationPluginManager(t *testing.T) { ctx := context.Background() verifyCorePluginCatalogue(t, ctx, ps) - verifyBundledPlugins(t, ctx, ps, reg) + verifyBundledPlugins(t, ctx, ps) verifyPluginStaticRoutes(t, ctx, ps, reg) verifyBackendProcesses(t, reg.Plugins(ctx)) verifyPluginQuery(t, ctx, client.ProvideService(reg, pCfg)) @@ -255,7 +253,7 @@ func verifyCorePluginCatalogue(t *testing.T, ctx context.Context, ps *store.Serv require.Equal(t, len(expPanels)+len(expDataSources)+len(expApps), len(ps.Plugins(ctx))) } -func verifyBundledPlugins(t *testing.T, ctx context.Context, ps *store.Service, reg registry.Service) { +func verifyBundledPlugins(t *testing.T, ctx context.Context, ps *store.Service) { t.Helper() dsPlugins := make(map[string]struct{}) @@ -268,9 +266,6 @@ func verifyBundledPlugins(t *testing.T, ctx context.Context, ps *store.Service, require.NotEqual(t, plugins.PluginDTO{}, inputPlugin) require.NotNil(t, dsPlugins["input"]) - intInputPlugin, exists := reg.Plugin(ctx, "input") - require.True(t, exists) - pluginRoutes := make(map[string]*plugins.StaticRoute) for _, r := range ps.Routes() { pluginRoutes[r.PluginID] = r @@ -278,7 +273,7 @@ func verifyBundledPlugins(t *testing.T, ctx context.Context, ps *store.Service, for _, pluginID := range []string{"input"} { require.Contains(t, pluginRoutes, pluginID) - require.True(t, strings.HasPrefix(pluginRoutes[pluginID].Directory, intInputPlugin.PluginDir)) + require.Equal(t, pluginRoutes[pluginID].Directory, inputPlugin.Base()) } } @@ -292,11 +287,11 @@ func verifyPluginStaticRoutes(t *testing.T, ctx context.Context, rr plugins.Stat inputPlugin, _ := reg.Plugin(ctx, "input") require.NotNil(t, routes["input"]) - require.Equal(t, routes["input"].Directory, inputPlugin.PluginDir) + require.Equal(t, routes["input"].Directory, inputPlugin.FS.Base()) testAppPlugin, _ := reg.Plugin(ctx, "test-app") require.Contains(t, routes, "test-app") - require.Equal(t, routes["test-app"].Directory, testAppPlugin.PluginDir) + require.Equal(t, routes["test-app"].Directory, testAppPlugin.FS.Base()) } func verifyBackendProcesses(t *testing.T, ps []*plugins.Plugin) { diff --git a/pkg/plugins/manager/signature/manifest.go b/pkg/plugins/manager/signature/manifest.go index a81216b2184..82061863827 100644 --- a/pkg/plugins/manager/signature/manifest.go +++ b/pkg/plugins/manager/signature/manifest.go @@ -11,7 +11,6 @@ import ( "net/url" "os" "path" - "path/filepath" "runtime" "strings" @@ -57,8 +56,8 @@ N1c5v9v/4h6qeA== var runningWindows = runtime.GOOS == "windows" -// pluginManifest holds details for the file manifest -type pluginManifest struct { +// PluginManifest holds details for the file manifest +type PluginManifest struct { Plugin string `json:"plugin"` Version string `json:"version"` KeyID string `json:"keyId"` @@ -73,20 +72,20 @@ type pluginManifest struct { RootURLs []string `json:"rootUrls"` } -func (m *pluginManifest) isV2() bool { +func (m *PluginManifest) isV2() bool { return strings.HasPrefix(m.ManifestVersion, "2.") } // readPluginManifest attempts to read and verify the plugin manifest // if any error occurs or the manifest is not valid, this will return an error -func readPluginManifest(body []byte) (*pluginManifest, error) { +func ReadPluginManifest(body []byte) (*PluginManifest, error) { block, _ := clearsign.Decode(body) if block == nil { return nil, errors.New("unable to decode manifest") } // Convert to a well typed object - var manifest pluginManifest + var manifest PluginManifest err := json.Unmarshal(block.Plaintext, &manifest) if err != nil { return nil, fmt.Errorf("%v: %w", "Error parsing manifest JSON", err) @@ -99,32 +98,54 @@ func readPluginManifest(body []byte) (*pluginManifest, error) { return &manifest, nil } -func Calculate(mlog log.Logger, plugin *plugins.Plugin) (plugins.Signature, error) { - if plugin.IsCorePlugin() { +func Calculate(mlog log.Logger, class plugins.Class, plugin plugins.FoundPlugin) (plugins.Signature, error) { + if class == plugins.Core { return plugins.Signature{ Status: plugins.SignatureInternal, }, nil } - pluginFiles, err := pluginFilesRequiringVerification(plugin) - if err != nil { - mlog.Warn("Could not collect plugin file information in directory", "pluginID", plugin.ID, "dir", plugin.PluginDir) + if len(plugin.FS.Files()) == 0 { + mlog.Warn("No plugin file information in directory", "pluginID", plugin.JSONData.ID) return plugins.Signature{ Status: plugins.SignatureInvalid, - }, err + }, nil } - byteValue := plugin.Manifest() + f, err := plugin.FS.Open("MANIFEST.txt") + if err != nil { + if errors.Is(err, plugins.ErrFileNotExist) { + mlog.Debug("Could not find a MANIFEST.txt", "id", plugin.JSONData.ID, "err", err) + return plugins.Signature{ + Status: plugins.SignatureUnsigned, + }, nil + } + + mlog.Debug("Could not open MANIFEST.txt", "id", plugin.JSONData.ID, "err", err) + return plugins.Signature{ + Status: plugins.SignatureInvalid, + }, nil + } + defer func() { + if f == nil { + return + } + if err = f.Close(); err != nil { + mlog.Warn("Failed to close plugin MANIFEST file", "err", err) + } + }() + + byteValue, err := io.ReadAll(f) if err != nil || len(byteValue) < 10 { - mlog.Debug("Plugin is unsigned", "id", plugin.ID) + mlog.Debug("MANIFEST.TXT is invalid", "id", plugin.JSONData.ID) return plugins.Signature{ Status: plugins.SignatureUnsigned, }, nil } - manifest, err := readPluginManifest(byteValue) + manifest, err := ReadPluginManifest(byteValue) if err != nil { - mlog.Debug("Plugin signature invalid", "id", plugin.ID, "err", err) + mlog.Debug("Plugin signature invalid", "id", plugin.JSONData.ID, "err", err) return plugins.Signature{ Status: plugins.SignatureInvalid, }, nil @@ -137,7 +158,7 @@ func Calculate(mlog log.Logger, plugin *plugins.Plugin) (plugins.Signature, erro } // Make sure the versions all match - if manifest.Plugin != plugin.ID || manifest.Version != plugin.Info.Version { + if manifest.Plugin != plugin.JSONData.ID || manifest.Version != plugin.JSONData.Info.Version { return plugins.Signature{ Status: plugins.SignatureModified, }, nil @@ -146,10 +167,10 @@ func Calculate(mlog log.Logger, plugin *plugins.Plugin) (plugins.Signature, erro // Validate that plugin is running within defined root URLs if len(manifest.RootURLs) > 0 { if match, err := urlMatch(manifest.RootURLs, setting.AppUrl, manifest.SignatureType); err != nil { - mlog.Warn("Could not verify if root URLs match", "plugin", plugin.ID, "rootUrls", manifest.RootURLs) + mlog.Warn("Could not verify if root URLs match", "plugin", plugin.JSONData.ID, "rootUrls", manifest.RootURLs) return plugins.Signature{}, err } else if !match { - mlog.Warn("Could not find root URL that matches running application URL", "plugin", plugin.ID, + mlog.Warn("Could not find root URL that matches running application URL", "plugin", plugin.JSONData.ID, "appUrl", setting.AppUrl, "rootUrls", manifest.RootURLs) return plugins.Signature{ Status: plugins.SignatureInvalid, @@ -161,7 +182,7 @@ func Calculate(mlog log.Logger, plugin *plugins.Plugin) (plugins.Signature, erro // Verify the manifest contents for p, hash := range manifest.Files { - err = verifyHash(mlog, plugin.ID, filepath.Join(plugin.PluginDir, p), hash) + err = verifyHash(mlog, plugin, p, hash) if err != nil { return plugins.Signature{ Status: plugins.SignatureModified, @@ -173,20 +194,28 @@ func Calculate(mlog log.Logger, plugin *plugins.Plugin) (plugins.Signature, erro // Track files missing from the manifest var unsignedFiles []string - for _, f := range pluginFiles { + for _, f := range plugin.FS.Files() { + // Ignoring unsigned Chromium debug.log so it doesn't invalidate the signature for Renderer plugin running on Windows + if runningWindows && plugin.JSONData.Type == plugins.Renderer && f == "chrome-win/debug.log" { + continue + } + + if f == "MANIFEST.txt" { + continue + } if _, exists := manifestFiles[f]; !exists { unsignedFiles = append(unsignedFiles, f) } } if len(unsignedFiles) > 0 { - mlog.Warn("The following files were not included in the signature", "plugin", plugin.ID, "files", unsignedFiles) + mlog.Warn("The following files were not included in the signature", "plugin", plugin.JSONData.ID, "files", unsignedFiles) return plugins.Signature{ Status: plugins.SignatureModified, }, nil } - mlog.Debug("Plugin signature valid", "id", plugin.ID) + mlog.Debug("Plugin signature valid", "id", plugin.JSONData.ID) return plugins.Signature{ Status: plugins.SignatureValid, Type: manifest.SignatureType, @@ -194,17 +223,17 @@ func Calculate(mlog log.Logger, plugin *plugins.Plugin) (plugins.Signature, erro }, nil } -func verifyHash(mlog log.Logger, pluginID string, path string, hash string) error { +func verifyHash(mlog log.Logger, plugin plugins.FoundPlugin, path, hash string) error { // nolint:gosec // We can ignore the gosec G304 warning on this one because `path` is based // on the path provided in a manifest file for a plugin and not user input. - f, err := os.Open(path) + f, err := plugin.FS.Open(path) if err != nil { if os.IsPermission(err) { - mlog.Warn("Could not open plugin file due to lack of permissions", "plugin", pluginID, "path", path) + mlog.Warn("Could not open plugin file due to lack of permissions", "plugin", plugin.JSONData.ID, "path", path) return errors.New("permission denied when attempting to read plugin file") } - mlog.Warn("Plugin file listed in the manifest was not found", "plugin", pluginID, "path", path) + mlog.Warn("Plugin file listed in the manifest was not found", "plugin", plugin.JSONData.ID, "path", path) return errors.New("plugin file listed in the manifest was not found") } defer func() { @@ -219,75 +248,13 @@ func verifyHash(mlog log.Logger, pluginID string, path string, hash string) erro } sum := hex.EncodeToString(h.Sum(nil)) if sum != hash { - mlog.Warn("Plugin file checksum does not match signature checksum", "plugin", pluginID, "path", path) + mlog.Warn("Plugin file checksum does not match signature checksum", "plugin", plugin.JSONData.ID, "path", path) return errors.New("plugin file checksum does not match signature checksum") } return nil } -// pluginFilesRequiringVerification gets plugin filenames that require verification for plugin signing -// returns filenames as a slice of posix style paths relative to plugin directory -func pluginFilesRequiringVerification(plugin *plugins.Plugin) ([]string, error) { - var files []string - err := filepath.Walk(plugin.PluginDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if info.Mode()&os.ModeSymlink == os.ModeSymlink { - symlinkPath, err := filepath.EvalSymlinks(path) - if err != nil { - return err - } - - symlink, err := os.Stat(symlinkPath) - if err != nil { - return err - } - - // verify that symlinked file is within plugin directory - p, err := filepath.Rel(plugin.PluginDir, symlinkPath) - if err != nil { - return err - } - if p == ".." || strings.HasPrefix(p, ".."+string(filepath.Separator)) { - return fmt.Errorf("file '%s' not inside of plugin directory", p) - } - - // skip adding symlinked directories - if symlink.IsDir() { - return nil - } - } - - // skip directories and MANIFEST.txt - if info.IsDir() || info.Name() == "MANIFEST.txt" { - return nil - } - - // Ignoring unsigned Chromium debug.log so it doesn't invalidate the signature for Renderer plugin running on Windows - if runningWindows && plugin.IsRenderer() && strings.HasSuffix(path, filepath.Join("chrome-win", "debug.log")) { - return nil - } - - // verify that file is within plugin directory - file, err := filepath.Rel(plugin.PluginDir, path) - if err != nil { - return err - } - if strings.HasPrefix(file, ".."+string(filepath.Separator)) { - return fmt.Errorf("file '%s' not inside of plugin directory", file) - } - - files = append(files, filepath.ToSlash(file)) - - return nil - }) - - return files, err -} - func urlMatch(specs []string, target string, signatureType plugins.SignatureType) (bool, error) { targetURL, err := url.Parse(target) if err != nil { @@ -328,7 +295,7 @@ func (r invalidFieldErr) Error() string { return fmt.Sprintf("valid manifest field %s is required", r.field) } -func validateManifest(m pluginManifest, block *clearsign.Block) error { +func validateManifest(m PluginManifest, block *clearsign.Block) error { if len(m.Plugin) == 0 { return invalidFieldErr{field: "plugin"} } diff --git a/pkg/plugins/manager/signature/manifest_test.go b/pkg/plugins/manager/signature/manifest_test.go index a0591051aa1..de257b59ed5 100644 --- a/pkg/plugins/manager/signature/manifest_test.go +++ b/pkg/plugins/manager/signature/manifest_test.go @@ -46,7 +46,7 @@ NR7DnB0CCQHO+4FlSPtXFTzNepoc+CytQyDAeOLMLmf2Tqhk2YShk+G/YlVX -----END PGP SIGNATURE-----` t.Run("valid manifest", func(t *testing.T) { - manifest, err := readPluginManifest([]byte(txt)) + manifest, err := ReadPluginManifest([]byte(txt)) require.NoError(t, err) require.NotNil(t, manifest) @@ -62,7 +62,7 @@ NR7DnB0CCQHO+4FlSPtXFTzNepoc+CytQyDAeOLMLmf2Tqhk2YShk+G/YlVX t.Run("invalid manifest", func(t *testing.T) { modified := strings.ReplaceAll(txt, "README.md", "xxxxxxxxxx") - _, err := readPluginManifest([]byte(modified)) + _, err := ReadPluginManifest([]byte(modified)) require.Error(t, err) }) } @@ -99,7 +99,7 @@ khdr/tZ1PDgRxMqB/u+Vtbpl0xSxgblnrDOYMSI= -----END PGP SIGNATURE-----` t.Run("valid manifest", func(t *testing.T) { - manifest, err := readPluginManifest([]byte(txt)) + manifest, err := ReadPluginManifest([]byte(txt)) require.NoError(t, err) require.NotNil(t, manifest) @@ -151,15 +151,18 @@ func TestCalculate(t *testing.T) { }) setting.AppUrl = tc.appURL - sig, err := Calculate(log.NewTestLogger(), &plugins.Plugin{ + basePath := filepath.Join(parentDir, "testdata/non-pvt-with-root-url/plugin") + sig, err := Calculate(log.NewTestLogger(), plugins.External, plugins.FoundPlugin{ JSONData: plugins.JSONData{ ID: "test-datasource", Info: plugins.Info{ Version: "1.0.0", }, }, - PluginDir: filepath.Join(parentDir, "testdata/non-pvt-with-root-url/plugin"), - Class: plugins.External, + FS: plugins.NewLocalFS(map[string]struct{}{ + filepath.Join(basePath, "MANIFEST.txt"): {}, + filepath.Join(basePath, "plugin.json"): {}, + }, basePath), }) require.NoError(t, err) require.Equal(t, tc.expectedSignature, sig) @@ -172,8 +175,10 @@ func TestCalculate(t *testing.T) { runningWindows = backup }) + basePath := "../testdata/renderer-added-file/plugin" + runningWindows = true - sig, err := Calculate(log.NewTestLogger(), &plugins.Plugin{ + sig, err := Calculate(log.NewTestLogger(), plugins.External, plugins.FoundPlugin{ JSONData: plugins.JSONData{ ID: "test-renderer", Type: plugins.Renderer, @@ -181,7 +186,11 @@ func TestCalculate(t *testing.T) { Version: "1.0.0", }, }, - PluginDir: "../testdata/renderer-added-file/plugin", + FS: plugins.NewLocalFS(map[string]struct{}{ + filepath.Join(basePath, "MANIFEST.txt"): {}, + filepath.Join(basePath, "plugin.json"): {}, + filepath.Join(basePath, "chrome-win/debug.log"): {}, + }, basePath), }) require.NoError(t, err) require.Equal(t, plugins.Signature{ @@ -192,7 +201,7 @@ func TestCalculate(t *testing.T) { }) } -func fileList(manifest *pluginManifest) []string { +func fileList(manifest *PluginManifest) []string { var keys []string for k := range manifest.Files { keys = append(keys, k) @@ -476,52 +485,52 @@ func Test_urlMatch_private(t *testing.T) { func Test_validateManifest(t *testing.T) { tcs := []struct { name string - manifest *pluginManifest + manifest *PluginManifest expectedErr string }{ { name: "Empty plugin field", - manifest: createV2Manifest(t, func(m *pluginManifest) { m.Plugin = "" }), + manifest: createV2Manifest(t, func(m *PluginManifest) { m.Plugin = "" }), expectedErr: "valid manifest field plugin is required", }, { name: "Empty keyId field", - manifest: createV2Manifest(t, func(m *pluginManifest) { m.KeyID = "" }), + manifest: createV2Manifest(t, func(m *PluginManifest) { m.KeyID = "" }), expectedErr: "valid manifest field keyId is required", }, { name: "Empty signedByOrg field", - manifest: createV2Manifest(t, func(m *pluginManifest) { m.SignedByOrg = "" }), + manifest: createV2Manifest(t, func(m *PluginManifest) { m.SignedByOrg = "" }), expectedErr: "valid manifest field signedByOrg is required", }, { name: "Empty signedByOrgName field", - manifest: createV2Manifest(t, func(m *pluginManifest) { m.SignedByOrgName = "" }), + manifest: createV2Manifest(t, func(m *PluginManifest) { m.SignedByOrgName = "" }), expectedErr: "valid manifest field SignedByOrgName is required", }, { name: "Empty signatureType field", - manifest: createV2Manifest(t, func(m *pluginManifest) { m.SignatureType = "" }), + manifest: createV2Manifest(t, func(m *PluginManifest) { m.SignatureType = "" }), expectedErr: "valid manifest field signatureType is required", }, { name: "Invalid signatureType field", - manifest: createV2Manifest(t, func(m *pluginManifest) { m.SignatureType = "invalidSignatureType" }), + manifest: createV2Manifest(t, func(m *PluginManifest) { m.SignatureType = "invalidSignatureType" }), expectedErr: "valid manifest field signatureType is required", }, { name: "Empty files field", - manifest: createV2Manifest(t, func(m *pluginManifest) { m.Files = map[string]string{} }), + manifest: createV2Manifest(t, func(m *PluginManifest) { m.Files = map[string]string{} }), expectedErr: "valid manifest field files is required", }, { name: "Empty time field", - manifest: createV2Manifest(t, func(m *pluginManifest) { m.Time = 0 }), + manifest: createV2Manifest(t, func(m *PluginManifest) { m.Time = 0 }), expectedErr: "valid manifest field time is required", }, { name: "Empty version field", - manifest: createV2Manifest(t, func(m *pluginManifest) { m.Version = "" }), + manifest: createV2Manifest(t, func(m *PluginManifest) { m.Version = "" }), expectedErr: "valid manifest field version is required", }, } @@ -533,10 +542,10 @@ func Test_validateManifest(t *testing.T) { } } -func createV2Manifest(t *testing.T, cbs ...func(*pluginManifest)) *pluginManifest { +func createV2Manifest(t *testing.T, cbs ...func(*PluginManifest)) *PluginManifest { t.Helper() - m := &pluginManifest{ + m := &PluginManifest{ Plugin: "grafana-test-app", Version: "2.5.3", KeyID: "7e4d0c6a708866e7", diff --git a/pkg/plugins/manager/sources/sources.go b/pkg/plugins/manager/sources/sources.go index 1e228ee45b0..f4c6d37a2c0 100644 --- a/pkg/plugins/manager/sources/sources.go +++ b/pkg/plugins/manager/sources/sources.go @@ -4,9 +4,9 @@ import ( "context" "path/filepath" - "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins/config" + "github.com/grafana/grafana/pkg/plugins/log" "github.com/grafana/grafana/pkg/setting" ) diff --git a/pkg/plugins/manager/store/store.go b/pkg/plugins/manager/store/store.go index 872a70a4866..e4f5f6bb58e 100644 --- a/pkg/plugins/manager/store/store.go +++ b/pkg/plugins/manager/store/store.go @@ -18,8 +18,9 @@ type Service struct { func ProvideService(pluginRegistry registry.Service, pluginSources sources.Resolver, pluginLoader loader.Service) (*Service, error) { - for _, ps := range pluginSources.List(context.Background()) { - if _, err := pluginLoader.Load(context.Background(), ps.Class, ps.Paths); err != nil { + ctx := context.Background() + for _, ps := range pluginSources.List(ctx) { + if _, err := pluginLoader.Load(ctx, ps.Class, ps.Paths); err != nil { return nil, err } } diff --git a/pkg/plugins/manager/store/store_test.go b/pkg/plugins/manager/store/store_test.go index 906c7db7e68..f992b0b4e46 100644 --- a/pkg/plugins/manager/store/store_test.go +++ b/pkg/plugins/manager/store/store_test.go @@ -97,11 +97,11 @@ func TestStore_Plugins(t *testing.T) { func TestStore_Routes(t *testing.T) { t.Run("Routes returns all static routes for non-decommissioned plugins", func(t *testing.T) { - p1 := &plugins.Plugin{JSONData: plugins.JSONData{ID: "a-test-renderer", Type: plugins.Renderer}, PluginDir: "/some/dir"} - p2 := &plugins.Plugin{JSONData: plugins.JSONData{ID: "b-test-panel", Type: plugins.Panel}, PluginDir: "/grafana/"} - p3 := &plugins.Plugin{JSONData: plugins.JSONData{ID: "c-test-secrets", Type: plugins.SecretsManager}, PluginDir: "./secrets", Class: plugins.Core} - p4 := &plugins.Plugin{JSONData: plugins.JSONData{ID: "d-test-datasource", Type: plugins.DataSource}, PluginDir: "../test"} - p5 := &plugins.Plugin{JSONData: plugins.JSONData{ID: "e-test-app", Type: plugins.App}} + p1 := &plugins.Plugin{JSONData: plugins.JSONData{ID: "a-test-renderer", Type: plugins.Renderer}, FS: fakes.NewFakePluginFiles("/some/dir")} + p2 := &plugins.Plugin{JSONData: plugins.JSONData{ID: "b-test-panel", Type: plugins.Panel}, FS: fakes.NewFakePluginFiles("/grafana/")} + p3 := &plugins.Plugin{JSONData: plugins.JSONData{ID: "c-test-secrets", Type: plugins.SecretsManager}, FS: fakes.NewFakePluginFiles("./secrets"), Class: plugins.Core} + p4 := &plugins.Plugin{JSONData: plugins.JSONData{ID: "d-test-datasource", Type: plugins.DataSource}, FS: fakes.NewFakePluginFiles("../test")} + p5 := &plugins.Plugin{JSONData: plugins.JSONData{ID: "e-test-app", Type: plugins.App}, FS: fakes.NewFakePluginFiles("any/path")} p6 := &plugins.Plugin{JSONData: plugins.JSONData{ID: "f-test-app", Type: plugins.App}} p6.RegisterClient(&DecommissionedPlugin{}) @@ -115,7 +115,7 @@ func TestStore_Routes(t *testing.T) { })) sr := func(p *plugins.Plugin) *plugins.StaticRoute { - return &plugins.StaticRoute{PluginID: p.ID, Directory: p.PluginDir} + return &plugins.StaticRoute{PluginID: p.ID, Directory: p.FS.Base()} } rs := ps.Routes() diff --git a/pkg/plugins/plugins.go b/pkg/plugins/plugins.go index ff389891a52..cb6310a3851 100644 --- a/pkg/plugins/plugins.go +++ b/pkg/plugins/plugins.go @@ -6,8 +6,7 @@ import ( "errors" "fmt" "io/fs" - "os" - "path/filepath" + "path" "runtime" "strings" @@ -26,8 +25,8 @@ var ErrFileNotExist = errors.New("file does not exist") type Plugin struct { JSONData - PluginDir string - Class Class + FS FS + Class Class // App fields IncludedInAppID string @@ -55,8 +54,8 @@ type Plugin struct { type PluginDTO struct { JSONData + fs FS logger log.Logger - pluginDir string supportsStreaming bool Class Class @@ -81,6 +80,10 @@ func (p PluginDTO) SupportsStreaming() bool { return p.supportsStreaming } +func (p PluginDTO) Base() string { + return p.fs.Base() +} + func (p PluginDTO) IsApp() bool { return p.Type == App } @@ -96,21 +99,15 @@ func (p PluginDTO) File(name string) (fs.File, error) { return nil, err } - absPluginDir, err := filepath.Abs(p.pluginDir) + if p.fs == nil { + return nil, ErrFileNotExist + } + + f, err := p.fs.Open(cleanPath) if err != nil { return nil, err } - absFilePath := filepath.Join(absPluginDir, cleanPath) - // Wrapping in filepath.Clean to properly handle - // gosec G304 Potential file inclusion via variable rule. - f, err := os.Open(filepath.Clean(absFilePath)) - if err != nil { - if os.IsNotExist(err) { - return nil, ErrFileNotExist - } - return nil, err - } return f, nil } @@ -337,6 +334,18 @@ func (p *Plugin) Client() (PluginClient, bool) { } func (p *Plugin) ExecutablePath() string { + if p.IsRenderer() { + return p.executablePath("plugin_start") + } + + if p.IsSecretsManager() { + return p.executablePath("secrets_plugin_start") + } + + return p.executablePath(p.Executable) +} + +func (p *Plugin) executablePath(f string) string { os := strings.ToLower(runtime.GOOS) arch := runtime.GOARCH extension := "" @@ -344,15 +353,7 @@ func (p *Plugin) ExecutablePath() string { if os == "windows" { extension = ".exe" } - if p.IsRenderer() { - return filepath.Join(p.PluginDir, fmt.Sprintf("%s_%s_%s%s", "plugin_start", os, strings.ToLower(arch), extension)) - } - - if p.IsSecretsManager() { - return filepath.Join(p.PluginDir, fmt.Sprintf("%s_%s_%s%s", "secrets_plugin_start", os, strings.ToLower(arch), extension)) - } - - return filepath.Join(p.PluginDir, fmt.Sprintf("%s_%s_%s%s", p.Executable, os, strings.ToLower(arch), extension)) + return path.Join(p.FS.Base(), fmt.Sprintf("%s_%s_%s%s", f, os, strings.ToLower(arch), extension)) } type PluginClient interface { @@ -366,9 +367,10 @@ type PluginClient interface { func (p *Plugin) ToDTO() PluginDTO { return PluginDTO{ logger: p.Logger(), - pluginDir: p.PluginDir, - JSONData: p.JSONData, + fs: p.FS, + supportsStreaming: p.client != nil && p.client.(backend.StreamHandler) != nil, Class: p.Class, + JSONData: p.JSONData, IncludedInAppID: p.IncludedInAppID, DefaultNavURL: p.DefaultNavURL, Pinned: p.Pinned, @@ -378,7 +380,6 @@ func (p *Plugin) ToDTO() PluginDTO { SignatureError: p.SignatureError, Module: p.Module, BaseURL: p.BaseURL, - supportsStreaming: p.client != nil && p.client.(backend.StreamHandler) != nil, } } @@ -387,7 +388,11 @@ func (p *Plugin) StaticRoute() *StaticRoute { return nil } - return &StaticRoute{Directory: p.PluginDir, PluginID: p.ID} + if p.FS == nil { + return nil + } + + return &StaticRoute{Directory: p.FS.Base(), PluginID: p.ID} } func (p *Plugin) IsRenderer() bool { @@ -414,15 +419,6 @@ func (p *Plugin) IsExternalPlugin() bool { return p.Class == External } -func (p *Plugin) Manifest() []byte { - d, err := os.ReadFile(filepath.Join(p.PluginDir, "MANIFEST.txt")) - if err != nil { - return []byte{} - } - - return d -} - type Class string const ( diff --git a/pkg/services/updatechecker/plugins_test.go b/pkg/services/updatechecker/plugins_test.go index 6028e735e8d..8003f61a027 100644 --- a/pkg/services/updatechecker/plugins_test.go +++ b/pkg/services/updatechecker/plugins_test.go @@ -135,6 +135,7 @@ func TestPluginUpdateChecker_checkForUpdates(t *testing.T) { Info: plugins.Info{Version: "0.9.0"}, Type: plugins.DataSource, }, + Class: plugins.External, }, { JSONData: plugins.JSONData{ @@ -142,6 +143,7 @@ func TestPluginUpdateChecker_checkForUpdates(t *testing.T) { Info: plugins.Info{Version: "0.5.0"}, Type: plugins.App, }, + Class: plugins.External, }, { JSONData: plugins.JSONData{ @@ -149,14 +151,15 @@ func TestPluginUpdateChecker_checkForUpdates(t *testing.T) { Info: plugins.Info{Version: "2.5.7"}, Type: plugins.Panel, }, + Class: plugins.Bundled, }, { - Class: plugins.Core, JSONData: plugins.JSONData{ ID: "test-core-panel", Info: plugins.Info{Version: "0.0.1"}, Type: plugins.Panel, }, + Class: plugins.Core, }, }, },