diff --git a/pkg/cmd/grafana-cli/commands/commands.go b/pkg/cmd/grafana-cli/commands/commands.go index 4c20cd4f0c3..a7b71579d86 100644 --- a/pkg/cmd/grafana-cli/commands/commands.go +++ b/pkg/cmd/grafana-cli/commands/commands.go @@ -73,7 +73,7 @@ var pluginCommands = []cli.Command{ }, { Name: "list-remote", Usage: "list remote available plugins", - Action: runPluginCommand(listremoteCommand), + Action: runPluginCommand(listRemoteCommand), }, { Name: "list-versions", Usage: "list-versions ", diff --git a/pkg/cmd/grafana-cli/commands/commandstest/fake_api_client.go b/pkg/cmd/grafana-cli/commands/commandstest/fake_api_client.go new file mode 100644 index 00000000000..5b3ba9ecccd --- /dev/null +++ b/pkg/cmd/grafana-cli/commands/commandstest/fake_api_client.go @@ -0,0 +1,34 @@ +package commandstest + +import ( + "github.com/grafana/grafana/pkg/cmd/grafana-cli/models" +) + +type FakeGrafanaComClient struct { + GetPluginFunc func(pluginId, repoUrl string) (models.Plugin, error) + DownloadFileFunc func(pluginName, filePath, url string, checksum string) (content []byte, err error) + ListAllPluginsFunc func(repoUrl string) (models.PluginRepo, error) +} + +func (client *FakeGrafanaComClient) GetPlugin(pluginId, repoUrl string) (models.Plugin, error) { + if client.GetPluginFunc != nil { + return client.GetPluginFunc(pluginId, repoUrl) + } + + return models.Plugin{}, nil +} + +func (client *FakeGrafanaComClient) DownloadFile(pluginName, filePath, url string, checksum string) (content []byte, err error) { + if client.DownloadFileFunc != nil { + return client.DownloadFileFunc(pluginName, filePath, url, checksum) + } + + return make([]byte, 0), nil +} + +func (client *FakeGrafanaComClient) ListAllPlugins(repoUrl string) (models.PluginRepo, error) { + if client.ListAllPluginsFunc != nil { + return client.ListAllPluginsFunc(repoUrl) + } + return models.PluginRepo{}, nil +} diff --git a/pkg/cmd/grafana-cli/commands/commandstest/fake_commandLine.go b/pkg/cmd/grafana-cli/commands/commandstest/fake_commandLine.go index f7afc57fda2..7982dfed4de 100644 --- a/pkg/cmd/grafana-cli/commands/commandstest/fake_commandLine.go +++ b/pkg/cmd/grafana-cli/commands/commandstest/fake_commandLine.go @@ -2,6 +2,7 @@ package commandstest import ( "github.com/codegangsta/cli" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" ) type FakeFlagger struct { @@ -12,6 +13,7 @@ type FakeCommandLine struct { LocalFlags, GlobalFlags *FakeFlagger HelpShown, VersionShown bool CliArgs []string + Client utils.ApiClient } func (ff FakeFlagger) String(key string) string { @@ -105,3 +107,7 @@ func (fcli *FakeCommandLine) PluginDirectory() string { func (fcli *FakeCommandLine) PluginURL() string { return fcli.GlobalString("pluginUrl") } + +func (fcli *FakeCommandLine) ApiClient() utils.ApiClient { + return fcli.Client +} diff --git a/pkg/cmd/grafana-cli/commands/install_command.go b/pkg/cmd/grafana-cli/commands/install_command.go index ad77c39ba87..f2ba500a953 100644 --- a/pkg/cmd/grafana-cli/commands/install_command.go +++ b/pkg/cmd/grafana-cli/commands/install_command.go @@ -6,15 +6,17 @@ import ( "errors" "fmt" "io" - "io/ioutil" - "net/http" "os" "path" + "path/filepath" "regexp" + "runtime" "strings" "github.com/fatih/color" "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" + "github.com/grafana/grafana/pkg/util/errutil" + "golang.org/x/xerrors" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models" @@ -64,13 +66,23 @@ func installCommand(c utils.CommandLine) error { func InstallPlugin(pluginName, version string, c utils.CommandLine) error { pluginFolder := c.PluginDirectory() downloadURL := c.PluginURL() + isInternal := false + + var checksum string if downloadURL == "" { - plugin, err := s.GetPlugin(pluginName, c.RepoDirectory()) + if strings.HasPrefix(pluginName, "grafana-") { + // At this point the plugin download is going through grafana.com API and thus the name is validated. + // Checking for grafana prefix is how it is done there so no 3rd party plugin should have that prefix. + // You can supply custom plugin name and then set custom download url to 3rd party plugin but then that + // is up to the user to know what she is doing. + isInternal = true + } + plugin, err := c.ApiClient().GetPlugin(pluginName, c.RepoDirectory()) if err != nil { return err } - v, err := SelectVersion(plugin, version) + v, err := SelectVersion(&plugin, version) if err != nil { return err } @@ -81,7 +93,13 @@ func InstallPlugin(pluginName, version string, c utils.CommandLine) error { downloadURL = fmt.Sprintf("%s/%s/versions/%s/download", c.GlobalString("repo"), pluginName, - version) + version, + ) + + // Plugins which are downloaded just as sourcecode zipball from github do not have checksum + if v.Arch != nil { + checksum = v.Arch[osAndArchString()].Md5 + } } logger.Infof("installing %v @ %v\n", pluginName, version) @@ -89,9 +107,14 @@ func InstallPlugin(pluginName, version string, c utils.CommandLine) error { logger.Infof("into: %v\n", pluginFolder) logger.Info("\n") - err := downloadFile(pluginName, pluginFolder, downloadURL) + content, err := c.ApiClient().DownloadFile(pluginName, pluginFolder, downloadURL, checksum) if err != nil { - return err + return errutil.Wrap("Failed to download plugin archive", err) + } + + err = extractFiles(content, pluginName, pluginFolder, isInternal) + if err != nil { + return errutil.Wrap("Failed to extract plugin archive", err) } logger.Infof("%s Installed %s successfully \n", color.GreenString("✔"), pluginName) @@ -105,18 +128,61 @@ func InstallPlugin(pluginName, version string, c utils.CommandLine) error { return err } -func SelectVersion(plugin m.Plugin, version string) (m.Version, error) { +func osAndArchString() string { + osString := strings.ToLower(runtime.GOOS) + arch := runtime.GOARCH + return osString + "-" + arch +} + +func supportsCurrentArch(version *m.Version) bool { + if version.Arch == nil { + return true + } + for arch := range version.Arch { + if arch == osAndArchString() || arch == "any" { + return true + } + } + return false +} + +func latestSupportedVersion(plugin *m.Plugin) *m.Version { + for _, ver := range plugin.Versions { + if supportsCurrentArch(&ver) { + return &ver + } + } + return nil +} + +// SelectVersion returns latest version if none is specified or the specified version. If the version string is not +// matched to existing version it errors out. It also errors out if version that is matched is not available for current +// os and platform. +func SelectVersion(plugin *m.Plugin, version string) (*m.Version, error) { + var ver *m.Version if version == "" { - return plugin.Versions[0], nil + ver = &plugin.Versions[0] } for _, v := range plugin.Versions { if v.Version == version { - return v, nil + ver = &v } } - return m.Version{}, errors.New("Could not find the version you're looking for") + if ver == nil { + return nil, xerrors.New("Could not find the version you're looking for") + } + + latestForArch := latestSupportedVersion(plugin) + if latestForArch == nil { + return nil, xerrors.New("Plugin is not supported on your architecture and os.") + } + + if latestForArch.Version == ver.Version { + return ver, nil + } + return nil, xerrors.Errorf("Version you want is not supported on your architecture and os. Latest suitable version is %v", latestForArch.Version) } func RemoveGitBuildFromName(pluginName, filename string) string { @@ -124,57 +190,19 @@ func RemoveGitBuildFromName(pluginName, filename string) string { return r.ReplaceAllString(filename, pluginName+"/") } -var retryCount = 0 var permissionsDeniedMessage = "Could not create %s. Permission denied. Make sure you have write access to plugindir" -func downloadFile(pluginName, filePath, url string) (err error) { - defer func() { - if r := recover(); r != nil { - retryCount++ - if retryCount < 3 { - fmt.Println("Failed downloading. Will retry once.") - err = downloadFile(pluginName, filePath, url) - } else { - failure := fmt.Sprintf("%v", r) - if failure == "runtime error: makeslice: len out of range" { - err = fmt.Errorf("Corrupt http response from source. Please try again") - } else { - panic(r) - } - } - } - }() - - var bytes []byte - - if _, err := os.Stat(url); err == nil { - bytes, err = ioutil.ReadFile(url) - if err != nil { - return err - } - } else { - resp, err := http.Get(url) // #nosec - if err != nil { - return err - } - defer resp.Body.Close() - - bytes, err = ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - } - - return extractFiles(bytes, pluginName, filePath) -} - -func extractFiles(body []byte, pluginName string, filePath string) error { +func extractFiles(body []byte, pluginName string, filePath string, allowSymlinks bool) error { r, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) if err != nil { return err } for _, zf := range r.File { - newFile := path.Join(filePath, RemoveGitBuildFromName(pluginName, zf.Name)) + newFileName := RemoveGitBuildFromName(pluginName, zf.Name) + if !isPathSafe(newFileName, path.Join(filePath, pluginName)) { + return xerrors.Errorf("filepath: %v tries to write outside of plugin directory: %v. This can be a security risk.", zf.Name, path.Join(filePath, pluginName)) + } + newFile := path.Join(filePath, newFileName) if zf.FileInfo().IsDir() { err := os.Mkdir(newFile, 0755) @@ -182,25 +210,24 @@ func extractFiles(body []byte, pluginName string, filePath string) error { return fmt.Errorf(permissionsDeniedMessage, newFile) } } else { - fileMode := zf.Mode() + if isSymlink(zf) { + if !allowSymlinks { + logger.Errorf("%v: plugin archive contains symlink which is not allowed. Skipping \n", zf.Name) + continue + } + err = extractSymlink(zf, newFile) + if err != nil { + logger.Errorf("Failed to extract symlink: %v \n", err) + continue + } + } else { - if strings.HasSuffix(newFile, "_linux_amd64") || strings.HasSuffix(newFile, "_darwin_amd64") { - fileMode = os.FileMode(0755) + err = extractFile(zf, newFile) + if err != nil { + logger.Errorf("Failed to extract file: %v \n", err) + continue + } } - - dst, err := os.OpenFile(newFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileMode) - if permissionsError(err) { - return fmt.Errorf(permissionsDeniedMessage, newFile) - } - - src, err := zf.Open() - if err != nil { - logger.Errorf("Failed to extract file: %v", err) - } - - io.Copy(dst, src) - dst.Close() - src.Close() } } @@ -210,3 +237,63 @@ func extractFiles(body []byte, pluginName string, filePath string) error { func permissionsError(err error) bool { return err != nil && strings.Contains(err.Error(), "permission denied") } + +func isSymlink(file *zip.File) bool { + return file.Mode()&os.ModeSymlink == os.ModeSymlink +} + +func extractSymlink(file *zip.File, filePath string) error { + // symlink target is the contents of the file + src, err := file.Open() + if err != nil { + return errutil.Wrap("Failed to extract file", err) + } + buf := new(bytes.Buffer) + _, err = io.Copy(buf, src) + if err != nil { + return errutil.Wrap("Failed to copy symlink contents", err) + } + err = os.Symlink(strings.TrimSpace(buf.String()), filePath) + if err != nil { + return errutil.Wrapf(err, "failed to make symbolic link for %v", filePath) + } + return nil +} + +func extractFile(file *zip.File, filePath string) (err error) { + fileMode := file.Mode() + // This is entry point for backend plugins so we want to make them executable + if strings.HasSuffix(filePath, "_linux_amd64") || strings.HasSuffix(filePath, "_darwin_amd64") { + fileMode = os.FileMode(0755) + } + + dst, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileMode) + if err != nil { + if permissionsError(err) { + return xerrors.Errorf(permissionsDeniedMessage, filePath) + } + return errutil.Wrap("Failed to open file", err) + } + defer func() { + err = dst.Close() + }() + + src, err := file.Open() + if err != nil { + return errutil.Wrap("Failed to extract file", err) + } + defer func() { + err = src.Close() + }() + + _, err = io.Copy(dst, src) + return +} + +// isPathSafe checks if the filePath does not resolve outside of destination. This is used to prevent +// https://snyk.io/research/zip-slip-vulnerability +// Based on https://github.com/mholt/archiver/pull/65/files#diff-635e4219ee55ef011b2b32bba065606bR109 +func isPathSafe(filePath string, destination string) bool { + destpath := filepath.Join(destination, filePath) + return strings.HasPrefix(destpath, destination) +} diff --git a/pkg/cmd/grafana-cli/commands/install_command_test.go b/pkg/cmd/grafana-cli/commands/install_command_test.go index 3554dda82a9..5babecb8fe7 100644 --- a/pkg/cmd/grafana-cli/commands/install_command_test.go +++ b/pkg/cmd/grafana-cli/commands/install_command_test.go @@ -1,11 +1,17 @@ package commands import ( + "fmt" "io/ioutil" "os" + "runtime" "testing" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/commands/commandstest" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/models" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" ) func TestFoldernameReplacement(t *testing.T) { @@ -41,40 +47,145 @@ func TestFoldernameReplacement(t *testing.T) { } func TestExtractFiles(t *testing.T) { - Convey("Should preserve file permissions for plugin backend binaries for linux and darwin", t, func() { - err := os.RemoveAll("testdata/fake-plugins-dir") - So(err, ShouldBeNil) - - err = os.MkdirAll("testdata/fake-plugins-dir", 0774) - So(err, ShouldBeNil) + t.Run("Should preserve file permissions for plugin backend binaries for linux and darwin", func(t *testing.T) { + pluginDir, del := setupFakePluginsDir(t) + defer del() body, err := ioutil.ReadFile("testdata/grafana-simple-json-datasource-ec18fa4da8096a952608a7e4c7782b4260b41bcf.zip") - So(err, ShouldBeNil) + assert.Nil(t, err) - err = extractFiles(body, "grafana-simple-json-datasource", "testdata/fake-plugins-dir") - So(err, ShouldBeNil) + err = extractFiles(body, "grafana-simple-json-datasource", pluginDir, false) + assert.Nil(t, err) - //File in zip has permissions 777 - fileInfo, err := os.Stat("testdata/fake-plugins-dir/grafana-simple-json-datasource/simple-plugin_darwin_amd64") - So(err, ShouldBeNil) - So(fileInfo.Mode().String(), ShouldEqual, "-rwxr-xr-x") - - //File in zip has permission 664 - fileInfo, err = os.Stat("testdata/fake-plugins-dir/grafana-simple-json-datasource/simple-plugin_linux_amd64") - So(err, ShouldBeNil) - So(fileInfo.Mode().String(), ShouldEqual, "-rwxr-xr-x") - - //File in zip has permission 644 - fileInfo, err = os.Stat("testdata/fake-plugins-dir/grafana-simple-json-datasource/simple-plugin_windows_amd64.exe") - So(err, ShouldBeNil) - So(fileInfo.Mode().String(), ShouldEqual, "-rw-r--r--") + //File in zip has permissions 755 + fileInfo, err := os.Stat(pluginDir + "/grafana-simple-json-datasource/simple-plugin_darwin_amd64") + assert.Nil(t, err) + assert.Equal(t, "-rwxr-xr-x", fileInfo.Mode().String()) //File in zip has permission 755 - fileInfo, err = os.Stat("testdata/fake-plugins-dir/grafana-simple-json-datasource/non-plugin-binary") - So(err, ShouldBeNil) - So(fileInfo.Mode().String(), ShouldEqual, "-rwxr-xr-x") + fileInfo, err = os.Stat(pluginDir + "/grafana-simple-json-datasource/simple-plugin_linux_amd64") + assert.Nil(t, err) + assert.Equal(t, "-rwxr-xr-x", fileInfo.Mode().String()) - err = os.RemoveAll("testdata/fake-plugins-dir") - So(err, ShouldBeNil) + //File in zip has permission 644 + fileInfo, err = os.Stat(pluginDir + "/grafana-simple-json-datasource/simple-plugin_windows_amd64.exe") + assert.Nil(t, err) + assert.Equal(t, "-rw-r--r--", fileInfo.Mode().String()) + + //File in zip has permission 755 + fileInfo, err = os.Stat(pluginDir + "/grafana-simple-json-datasource/non-plugin-binary") + assert.Nil(t, err) + assert.Equal(t, "-rwxr-xr-x", fileInfo.Mode().String()) + }) + + t.Run("Should ignore symlinks if not allowed", func(t *testing.T) { + pluginDir, del := setupFakePluginsDir(t) + defer del() + + body, err := ioutil.ReadFile("testdata/plugin-with-symlink.zip") + assert.Nil(t, err) + + err = extractFiles(body, "plugin-with-symlink", pluginDir, false) + assert.Nil(t, err) + + _, err = os.Stat(pluginDir + "/plugin-with-symlink/text.txt") + assert.Nil(t, err) + _, err = os.Stat(pluginDir + "/plugin-with-symlink/symlink_to_txt") + assert.NotNil(t, err) + }) + + t.Run("Should extract symlinks if allowed", func(t *testing.T) { + pluginDir, del := setupFakePluginsDir(t) + defer del() + + body, err := ioutil.ReadFile("testdata/plugin-with-symlink.zip") + assert.Nil(t, err) + + err = extractFiles(body, "plugin-with-symlink", pluginDir, true) + assert.Nil(t, err) + + _, err = os.Stat(pluginDir + "/plugin-with-symlink/symlink_to_txt") + assert.Nil(t, err) + fmt.Println(err) }) } + +func TestInstallPluginCommand(t *testing.T) { + pluginDir, del := setupFakePluginsDir(t) + defer del() + cmd := setupPluginInstallCmd(t, pluginDir) + err := InstallPlugin("test-plugin-panel", "", cmd) + assert.Nil(t, err) +} + +func TestIsPathSafe(t *testing.T) { + t.Run("Should be true on nested destinations", func(t *testing.T) { + assert.True(t, isPathSafe("dest", "/test/path")) + assert.True(t, isPathSafe("dest/one", "/test/path")) + assert.True(t, isPathSafe("../path/dest/one", "/test/path")) + }) + + t.Run("Should be false on destinations outside of path", func(t *testing.T) { + assert.False(t, isPathSafe("../dest", "/test/path")) + assert.False(t, isPathSafe("../../", "/test/path")) + assert.False(t, isPathSafe("../../test", "/test/path")) + }) + +} + +func setupPluginInstallCmd(t *testing.T, pluginDir string) utils.CommandLine { + cmd := &commandstest.FakeCommandLine{ + GlobalFlags: &commandstest.FakeFlagger{Data: map[string]interface{}{ + "pluginsDir": pluginDir, + }}, + } + + client := &commandstest.FakeGrafanaComClient{} + + client.GetPluginFunc = func(pluginId, repoUrl string) (models.Plugin, error) { + assert.Equal(t, "test-plugin-panel", pluginId) + plugin := models.Plugin{ + Id: "test-plugin-panel", + Category: "", + Versions: []models.Version{ + { + Commit: "commit", + Url: "url", + Version: "1.0.0", + Arch: map[string]models.ArchMeta{ + fmt.Sprintf("%s-%s", runtime.GOOS, runtime.GOARCH): { + Md5: "test", + }, + }, + }, + }, + } + return plugin, nil + } + + client.DownloadFileFunc = func(pluginName, filePath, url string, checksum string) (content []byte, err error) { + assert.Equal(t, "test-plugin-panel", pluginName) + assert.Equal(t, "/test-plugin-panel/versions/1.0.0/download", url) + assert.Equal(t, "test", checksum) + body, err := ioutil.ReadFile("testdata/grafana-simple-json-datasource-ec18fa4da8096a952608a7e4c7782b4260b41bcf.zip") + assert.Nil(t, err) + return body, nil + } + + cmd.Client = client + return cmd +} + +func setupFakePluginsDir(t *testing.T) (string, func()) { + dirname := "testdata/fake-plugins-dir" + err := os.RemoveAll(dirname) + assert.Nil(t, err) + + err = os.MkdirAll(dirname, 0774) + assert.Nil(t, err) + + return dirname, func() { + err = os.RemoveAll(dirname) + assert.Nil(t, err) + } +} diff --git a/pkg/cmd/grafana-cli/commands/listremote_command.go b/pkg/cmd/grafana-cli/commands/listremote_command.go index 7351ee58a37..2f587386192 100644 --- a/pkg/cmd/grafana-cli/commands/listremote_command.go +++ b/pkg/cmd/grafana-cli/commands/listremote_command.go @@ -2,24 +2,26 @@ package commands import ( "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" - s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" ) -func listremoteCommand(c utils.CommandLine) error { - plugin, err := s.ListAllPlugins(c.RepoDirectory()) +// listRemoteCommand prints out all plugins in the remote repo with latest version supported on current platform. +// If there are no supported versions for plugin it is skipped. +func listRemoteCommand(c utils.CommandLine) error { + plugin, err := c.ApiClient().ListAllPlugins(c.RepoDirectory()) if err != nil { return err } - for _, i := range plugin.Plugins { - pluginVersion := "" - if len(i.Versions) > 0 { - pluginVersion = i.Versions[0].Version + for _, plugin := range plugin.Plugins { + if len(plugin.Versions) > 0 { + ver := latestSupportedVersion(&plugin) + if ver != nil { + logger.Infof("id: %v version: %s\n", plugin.Id, ver.Version) + } } - logger.Infof("id: %v version: %s\n", i.Id, pluginVersion) } return nil diff --git a/pkg/cmd/grafana-cli/commands/listversions_command.go b/pkg/cmd/grafana-cli/commands/listversions_command.go index 78d681c06a3..a47977903a7 100644 --- a/pkg/cmd/grafana-cli/commands/listversions_command.go +++ b/pkg/cmd/grafana-cli/commands/listversions_command.go @@ -4,7 +4,6 @@ import ( "errors" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" - s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" ) @@ -24,7 +23,7 @@ func listversionsCommand(c utils.CommandLine) error { pluginToList := c.Args().First() - plugin, err := s.GetPlugin(pluginToList, c.GlobalString("repo")) + plugin, err := c.ApiClient().GetPlugin(pluginToList, c.GlobalString("repo")) if err != nil { return err } diff --git a/pkg/cmd/grafana-cli/commands/testdata/plugin-with-symlink.zip b/pkg/cmd/grafana-cli/commands/testdata/plugin-with-symlink.zip new file mode 100644 index 00000000000..43f87b2b7a4 Binary files /dev/null and b/pkg/cmd/grafana-cli/commands/testdata/plugin-with-symlink.zip differ diff --git a/pkg/cmd/grafana-cli/commands/upgrade_all_command.go b/pkg/cmd/grafana-cli/commands/upgrade_all_command.go index a5aadbbb0c2..66b6e97f3d4 100644 --- a/pkg/cmd/grafana-cli/commands/upgrade_all_command.go +++ b/pkg/cmd/grafana-cli/commands/upgrade_all_command.go @@ -8,24 +8,18 @@ import ( "github.com/hashicorp/go-version" ) -func ShouldUpgrade(installed string, remote m.Plugin) bool { - installedVersion, err1 := version.NewVersion(installed) - - if err1 != nil { +func shouldUpgrade(installed string, remote *m.Plugin) bool { + installedVersion, err := version.NewVersion(installed) + if err != nil { return false } - for _, v := range remote.Versions { - remoteVersion, err2 := version.NewVersion(v.Version) - - if err2 == nil { - if installedVersion.LessThan(remoteVersion) { - return true - } - } + latest := latestSupportedVersion(remote) + latestVersion, err := version.NewVersion(latest.Version) + if err != nil { + return false } - - return false + return installedVersion.LessThan(latestVersion) } func upgradeAllCommand(c utils.CommandLine) error { @@ -33,7 +27,7 @@ func upgradeAllCommand(c utils.CommandLine) error { localPlugins := s.GetLocalPlugins(pluginsDir) - remotePlugins, err := s.ListAllPlugins(c.GlobalString("repo")) + remotePlugins, err := c.ApiClient().ListAllPlugins(c.GlobalString("repo")) if err != nil { return err @@ -44,7 +38,7 @@ func upgradeAllCommand(c utils.CommandLine) error { for _, localPlugin := range localPlugins { for _, remotePlugin := range remotePlugins.Plugins { if localPlugin.Id == remotePlugin.Id { - if ShouldUpgrade(localPlugin.Info.Version, remotePlugin) { + if shouldUpgrade(localPlugin.Info.Version, &remotePlugin) { pluginsToUpgrade = append(pluginsToUpgrade, localPlugin) } } diff --git a/pkg/cmd/grafana-cli/commands/upgrade_all_command_test.go b/pkg/cmd/grafana-cli/commands/upgrade_all_command_test.go index 235b6a3930b..eaf673b334f 100644 --- a/pkg/cmd/grafana-cli/commands/upgrade_all_command_test.go +++ b/pkg/cmd/grafana-cli/commands/upgrade_all_command_test.go @@ -1,46 +1,47 @@ package commands import ( + "fmt" "testing" - m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models" - . "github.com/smartystreets/goconvey/convey" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/models" + "github.com/stretchr/testify/assert" ) func TestVersionComparsion(t *testing.T) { - Convey("Validate that version is outdated", t, func() { - versions := []m.Version{ + t.Run("Validate that version is outdated", func(t *testing.T) { + versions := []models.Version{ {Version: "1.1.1"}, {Version: "2.0.0"}, } - shouldUpgrade := map[string]m.Plugin{ + upgradeablePlugins := map[string]models.Plugin{ "0.0.0": {Versions: versions}, "1.0.0": {Versions: versions}, } - Convey("should return error", func() { - for k, v := range shouldUpgrade { - So(ShouldUpgrade(k, v), ShouldBeTrue) - } - }) + for k, v := range upgradeablePlugins { + t.Run(fmt.Sprintf("for %s should be true", k), func(t *testing.T) { + assert.True(t, shouldUpgrade(k, &v)) + }) + } }) - Convey("Validate that version is ok", t, func() { - versions := []m.Version{ + t.Run("Validate that version is ok", func(t *testing.T) { + versions := []models.Version{ {Version: "1.1.1"}, {Version: "2.0.0"}, } - shouldNotUpgrade := map[string]m.Plugin{ + shouldNotUpgrade := map[string]models.Plugin{ "2.0.0": {Versions: versions}, "6.0.0": {Versions: versions}, } - Convey("should return error", func() { - for k, v := range shouldNotUpgrade { - So(ShouldUpgrade(k, v), ShouldBeFalse) - } - }) + for k, v := range shouldNotUpgrade { + t.Run(fmt.Sprintf("for %s should be false", k), func(t *testing.T) { + assert.False(t, shouldUpgrade(k, &v)) + }) + } }) } diff --git a/pkg/cmd/grafana-cli/commands/upgrade_command.go b/pkg/cmd/grafana-cli/commands/upgrade_command.go index f32961ce589..36a9e6e29bd 100644 --- a/pkg/cmd/grafana-cli/commands/upgrade_command.go +++ b/pkg/cmd/grafana-cli/commands/upgrade_command.go @@ -17,13 +17,13 @@ func upgradeCommand(c utils.CommandLine) error { return err } - v, err2 := s.GetPlugin(pluginName, c.RepoDirectory()) + plugin, err2 := c.ApiClient().GetPlugin(pluginName, c.RepoDirectory()) if err2 != nil { return err2 } - if ShouldUpgrade(localPlugin.Info.Version, v) { + if shouldUpgrade(localPlugin.Info.Version, &plugin) { s.RemoveInstalledPlugin(pluginsDir, pluginName) return InstallPlugin(pluginName, "", c) } diff --git a/pkg/cmd/grafana-cli/models/model.go b/pkg/cmd/grafana-cli/models/model.go index 0700cb9a9e4..9c1f8d0f3c8 100644 --- a/pkg/cmd/grafana-cli/models/model.go +++ b/pkg/cmd/grafana-cli/models/model.go @@ -33,6 +33,12 @@ type Version struct { Commit string `json:"commit"` Url string `json:"url"` Version string `json:"version"` + // os-arch to md5 checksum to check when downloading the file + Arch map[string]ArchMeta `json:"arch"` +} + +type ArchMeta struct { + Md5 string `json:"md5"` } type PluginRepo struct { diff --git a/pkg/cmd/grafana-cli/services/api_client.go b/pkg/cmd/grafana-cli/services/api_client.go new file mode 100644 index 00000000000..44ccca29f54 --- /dev/null +++ b/pkg/cmd/grafana-cli/services/api_client.go @@ -0,0 +1,160 @@ +package services + +import ( + "crypto/md5" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "path" + "runtime" + + "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/models" + "github.com/grafana/grafana/pkg/util/errutil" + "golang.org/x/xerrors" +) + +type GrafanaComClient struct { + retryCount int +} + +func (client *GrafanaComClient) GetPlugin(pluginId, repoUrl string) (models.Plugin, error) { + logger.Debugf("getting plugin metadata from: %v pluginId: %v \n", repoUrl, pluginId) + body, err := sendRequest(HttpClient, repoUrl, "repo", pluginId) + + if err != nil { + if err == ErrNotFoundError { + return models.Plugin{}, errutil.Wrap("Failed to find requested plugin, check if the plugin_id is correct", err) + } + return models.Plugin{}, errutil.Wrap("Failed to send request", err) + } + + var data models.Plugin + err = json.Unmarshal(body, &data) + if err != nil { + logger.Info("Failed to unmarshal plugin repo response error:", err) + return models.Plugin{}, err + } + + return data, nil +} + +func (client *GrafanaComClient) DownloadFile(pluginName, filePath, url string, checksum string) (content []byte, err error) { + // Try handling url like local file path first + if _, err := os.Stat(url); err == nil { + bytes, err := ioutil.ReadFile(url) + if err != nil { + return nil, errutil.Wrap("Failed to read file", err) + } + return bytes, nil + } + + client.retryCount = 0 + + defer func() { + if r := recover(); r != nil { + client.retryCount++ + if client.retryCount < 3 { + logger.Info("Failed downloading. Will retry once.") + content, err = client.DownloadFile(pluginName, filePath, url, checksum) + } else { + client.retryCount = 0 + failure := fmt.Sprintf("%v", r) + if failure == "runtime error: makeslice: len out of range" { + err = xerrors.New("Corrupt http response from source. Please try again") + } else { + panic(r) + } + } + } + }() + + // TODO: this would be better if it was streamed file by file instead of buffered. + // Using no timeout here as some plugins can be bigger and smaller timeout would prevent to download a plugin on + // slow network. As this is CLI operation hanging is not a big of an issue as user can just abort. + body, err := sendRequest(HttpClientNoTimeout, url) + + if err != nil { + return nil, errutil.Wrap("Failed to send request", err) + } + + if len(checksum) > 0 && checksum != fmt.Sprintf("%x", md5.Sum(body)) { + return nil, xerrors.New("Expected MD5 checksum does not match the downloaded archive. Please contact security@grafana.com.") + } + return body, nil +} + +func (client *GrafanaComClient) ListAllPlugins(repoUrl string) (models.PluginRepo, error) { + body, err := sendRequest(HttpClient, repoUrl, "repo") + + if err != nil { + logger.Info("Failed to send request", "error", err) + return models.PluginRepo{}, errutil.Wrap("Failed to send request", err) + } + + var data models.PluginRepo + err = json.Unmarshal(body, &data) + if err != nil { + logger.Info("Failed to unmarshal plugin repo response error:", err) + return models.PluginRepo{}, err + } + + return data, nil +} + +func sendRequest(client http.Client, repoUrl string, subPaths ...string) ([]byte, error) { + u, _ := url.Parse(repoUrl) + for _, v := range subPaths { + u.Path = path.Join(u.Path, v) + } + + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + + req.Header.Set("grafana-version", grafanaVersion) + req.Header.Set("grafana-os", runtime.GOOS) + req.Header.Set("grafana-arch", runtime.GOARCH) + req.Header.Set("User-Agent", "grafana "+grafanaVersion) + + if err != nil { + return []byte{}, err + } + + res, err := client.Do(req) + if err != nil { + return []byte{}, err + } + return handleResponse(res) +} + +func handleResponse(res *http.Response) ([]byte, error) { + if res.StatusCode == 404 { + return []byte{}, ErrNotFoundError + } + + if res.StatusCode/100 != 2 && res.StatusCode/100 != 4 { + return []byte{}, fmt.Errorf("Api returned invalid status: %s", res.Status) + } + + body, err := ioutil.ReadAll(res.Body) + defer res.Body.Close() + + if res.StatusCode/100 == 4 { + if len(body) == 0 { + return []byte{}, &BadRequestError{Status: res.Status} + } + var message string + var jsonBody map[string]string + err = json.Unmarshal(body, &jsonBody) + if err != nil || len(jsonBody["message"]) == 0 { + message = string(body) + } else { + message = jsonBody["message"] + } + return []byte{}, &BadRequestError{Status: res.Status, Message: message} + } + + return body, err +} diff --git a/pkg/cmd/grafana-cli/services/api_client_test.go b/pkg/cmd/grafana-cli/services/api_client_test.go new file mode 100644 index 00000000000..3d7a0575463 --- /dev/null +++ b/pkg/cmd/grafana-cli/services/api_client_test.go @@ -0,0 +1,67 @@ +package services + +import ( + "bytes" + "io" + "io/ioutil" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHandleResponse(t *testing.T) { + t.Run("Returns body if status == 200", func(t *testing.T) { + body, err := handleResponse(makeResponse(200, "test")) + assert.Nil(t, err) + assert.Equal(t, "test", string(body)) + }) + + t.Run("Returns ErrorNotFound if status == 404", func(t *testing.T) { + _, err := handleResponse(makeResponse(404, "")) + assert.Equal(t, ErrNotFoundError, err) + }) + + t.Run("Returns message from body if status == 400", func(t *testing.T) { + _, err := handleResponse(makeResponse(400, "{ \"message\": \"error_message\" }")) + assert.NotNil(t, err) + assert.Equal(t, "error_message", asBadRequestError(t, err).Message) + }) + + t.Run("Returns body if status == 400 and no message key", func(t *testing.T) { + _, err := handleResponse(makeResponse(400, "{ \"test\": \"test_message\"}")) + assert.NotNil(t, err) + assert.Equal(t, "{ \"test\": \"test_message\"}", asBadRequestError(t, err).Message) + }) + + t.Run("Returns Bad request error if status == 400 and no body", func(t *testing.T) { + _, err := handleResponse(makeResponse(400, "")) + assert.NotNil(t, err) + _ = asBadRequestError(t, err) + }) + + t.Run("Returns error with invalid status if status == 500", func(t *testing.T) { + _, err := handleResponse(makeResponse(500, "")) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "invalid status") + }) +} + +func makeResponse(status int, body string) *http.Response { + return &http.Response{ + StatusCode: status, + Body: makeBody(body), + } +} + +func makeBody(body string) io.ReadCloser { + return ioutil.NopCloser(bytes.NewReader([]byte(body))) +} + +func asBadRequestError(t *testing.T, err error) *BadRequestError { + if badRequestError, ok := err.(*BadRequestError); ok { + return badRequestError + } + assert.FailNow(t, "Error was not of type BadRequestError") + return nil +} diff --git a/pkg/cmd/grafana-cli/services/services.go b/pkg/cmd/grafana-cli/services/services.go index a0d9d082a37..518b042b1e3 100644 --- a/pkg/cmd/grafana-cli/services/services.go +++ b/pkg/cmd/grafana-cli/services/services.go @@ -5,12 +5,9 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "net" "net/http" - "net/url" "path" - "runtime" "time" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" @@ -18,15 +15,33 @@ import ( ) var ( - IoHelper m.IoUtil = IoUtilImp{} - HttpClient http.Client - grafanaVersion string - ErrNotFoundError = errors.New("404 not found error") + IoHelper m.IoUtil = IoUtilImp{} + HttpClient http.Client + HttpClientNoTimeout http.Client + grafanaVersion string + ErrNotFoundError = errors.New("404 not found error") ) +type BadRequestError struct { + Message string + Status string +} + +func (e *BadRequestError) Error() string { + if len(e.Message) > 0 { + return fmt.Sprintf("%s: %s", e.Status, e.Message) + } + return e.Status +} + func Init(version string, skipTLSVerify bool) { grafanaVersion = version + HttpClient = makeHttpClient(skipTLSVerify, 10*time.Second) + HttpClientNoTimeout = makeHttpClient(skipTLSVerify, 0) +} + +func makeHttpClient(skipTLSVerify bool, timeout time.Duration) http.Client { tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ @@ -42,30 +57,12 @@ func Init(version string, skipTLSVerify bool) { }, } - HttpClient = http.Client{ - Timeout: 10 * time.Second, + return http.Client{ + Timeout: timeout, Transport: tr, } } -func ListAllPlugins(repoUrl string) (m.PluginRepo, error) { - body, err := sendRequest(repoUrl, "repo") - - if err != nil { - logger.Info("Failed to send request", "error", err) - return m.PluginRepo{}, fmt.Errorf("Failed to send request. error: %v", err) - } - - var data m.PluginRepo - err = json.Unmarshal(body, &data) - if err != nil { - logger.Info("Failed to unmarshal plugin repo response error:", err) - return m.PluginRepo{}, err - } - - return data, nil -} - func ReadPlugin(pluginDir, pluginName string) (m.InstalledPlugin, error) { distPluginDataPath := path.Join(pluginDir, pluginName, "dist", "plugin.json") @@ -120,60 +117,3 @@ func RemoveInstalledPlugin(pluginPath, pluginName string) error { return IoHelper.RemoveAll(pluginDir) } - -func GetPlugin(pluginId, repoUrl string) (m.Plugin, error) { - logger.Debugf("getting plugin metadata from: %v pluginId: %v \n", repoUrl, pluginId) - body, err := sendRequest(repoUrl, "repo", pluginId) - - if err != nil { - logger.Info("Failed to send request: ", err) - if err == ErrNotFoundError { - return m.Plugin{}, fmt.Errorf("Failed to find requested plugin, check if the plugin_id is correct. error: %v", err) - } - return m.Plugin{}, fmt.Errorf("Failed to send request. error: %v", err) - } - - var data m.Plugin - err = json.Unmarshal(body, &data) - if err != nil { - logger.Info("Failed to unmarshal plugin repo response error:", err) - return m.Plugin{}, err - } - - return data, nil -} - -func sendRequest(repoUrl string, subPaths ...string) ([]byte, error) { - u, _ := url.Parse(repoUrl) - for _, v := range subPaths { - u.Path = path.Join(u.Path, v) - } - - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - - req.Header.Set("grafana-version", grafanaVersion) - req.Header.Set("grafana-os", runtime.GOOS) - req.Header.Set("grafana-arch", runtime.GOARCH) - req.Header.Set("User-Agent", "grafana "+grafanaVersion) - - if err != nil { - return []byte{}, err - } - - res, err := HttpClient.Do(req) - if err != nil { - return []byte{}, err - } - - if res.StatusCode == 404 { - return []byte{}, ErrNotFoundError - } - if res.StatusCode/100 != 2 { - return []byte{}, fmt.Errorf("Api returned invalid status: %s", res.Status) - } - - body, err := ioutil.ReadAll(res.Body) - defer res.Body.Close() - - return body, err -} diff --git a/pkg/cmd/grafana-cli/utils/command_line.go b/pkg/cmd/grafana-cli/utils/command_line.go index 15546f2f392..f7fbb9f1754 100644 --- a/pkg/cmd/grafana-cli/utils/command_line.go +++ b/pkg/cmd/grafana-cli/utils/command_line.go @@ -2,6 +2,8 @@ package utils import ( "github.com/codegangsta/cli" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/models" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" ) type CommandLine interface { @@ -20,6 +22,13 @@ type CommandLine interface { PluginDirectory() string RepoDirectory() string PluginURL() string + ApiClient() ApiClient +} + +type ApiClient interface { + GetPlugin(pluginId, repoUrl string) (models.Plugin, error) + DownloadFile(pluginName, filePath, url string, checksum string) (content []byte, err error) + ListAllPlugins(repoUrl string) (models.PluginRepo, error) } type ContextCommandLine struct { @@ -57,3 +66,7 @@ func (c *ContextCommandLine) PluginURL() string { func (c *ContextCommandLine) OptionsString() string { return c.GlobalString("configOverrides") } + +func (c *ContextCommandLine) ApiClient() ApiClient { + return &services.GrafanaComClient{} +} diff --git a/pkg/cmd/grafana-cli/utils/grafana_path.go b/pkg/cmd/grafana-cli/utils/grafana_path.go index 373120a50c4..96ecd7656c7 100644 --- a/pkg/cmd/grafana-cli/utils/grafana_path.go +++ b/pkg/cmd/grafana-cli/utils/grafana_path.go @@ -6,37 +6,52 @@ import ( "path/filepath" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" + "golang.org/x/xerrors" ) func GetGrafanaPluginDir(currentOS string) string { - if isDevEnvironment() { - return "../data/plugins" + if rootPath, ok := tryGetRootForDevEnvironment(); ok { + return filepath.Join(rootPath, "data/plugins") } return returnOsDefault(currentOS) } -func isDevEnvironment() bool { - // if ../conf/defaults.ini exists, grafana is not installed as package - // that its in development environment. +// getGrafanaRoot tries to get root of directory when developing grafana ie repo root. It is not perfect it just +// checks what is the binary path and tries to guess based on that but if it is not running in dev env you get a bogus +// path back. +func getGrafanaRoot() (string, error) { ex, err := os.Executable() if err != nil { - logger.Error("Could not get executable path. Assuming non dev environment.") - return false + return "", xerrors.New("Failed to get executable path") } exPath := filepath.Dir(ex) _, last := path.Split(exPath) if last == "bin" { // In dev env the executable for current platform is created in 'bin/' dir - defaultsPath := filepath.Join(exPath, "../conf/defaults.ini") - _, err = os.Stat(defaultsPath) - return err == nil + return filepath.Join(exPath, ".."), nil } // But at the same time there are per platform directories that contain the binaries and can also be used. - defaultsPath := filepath.Join(exPath, "../../conf/defaults.ini") + return filepath.Join(exPath, "../.."), nil +} + +// tryGetRootForDevEnvironment returns root path if we are in dev environment. It checks if conf/defaults.ini exists +// which should only exist in dev. Second param is false if we are not in dev or if it wasn't possible to determine it. +func tryGetRootForDevEnvironment() (string, bool) { + rootPath, err := getGrafanaRoot() + if err != nil { + logger.Error("Could not get executable path. Assuming non dev environment.", err) + return "", false + } + + defaultsPath := filepath.Join(rootPath, "conf/defaults.ini") + _, err = os.Stat(defaultsPath) - return err == nil + if err != nil { + return "", false + } + return rootPath, true } func returnOsDefault(currentOs string) string { diff --git a/pkg/plugins/plugins.go b/pkg/plugins/plugins.go index 4c75e6b3d58..8afbe870c14 100644 --- a/pkg/plugins/plugins.go +++ b/pkg/plugins/plugins.go @@ -164,11 +164,15 @@ func scan(pluginDir string) error { } func (scanner *PluginScanner) walker(currentPath string, f os.FileInfo, err error) error { + // We scan all the subfolders for plugin.json (with some exceptions) so that we also load embedded plugins, for + // example https://github.com/raintank/worldping-app/tree/master/dist/grafana-worldmap-panel worldmap panel plugin + // is embedded in worldping app. + if err != nil { return err } - if f.Name() == "node_modules" { + if f.Name() == "node_modules" || f.Name() == "Chromium.app" { return util.ErrWalkSkipDir } diff --git a/pkg/services/rendering/rendering.go b/pkg/services/rendering/rendering.go index 259b5db2d7a..8d6605645fd 100644 --- a/pkg/services/rendering/rendering.go +++ b/pkg/services/rendering/rendering.go @@ -67,6 +67,7 @@ func (rs *RenderingService) Run(ctx context.Context) error { } if plugins.Renderer == nil { + rs.log.Info("Backend rendering via phantomJS") rs.renderAction = rs.renderViaPhantomJS <-ctx.Done() return nil